ghtorrent 0.9 → 0.10

data/lib/ghtorrent/migrations/017_drop_forks_table.rb

@@ -0,0 +1,24 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+    puts 'Dropping table forks'
+    drop_table :forks
+  end
+
+  down do
+    puts 'Adding table forks'
+
+    create_table :forks do
+      foreign_key :forked_project_id, :projects, :null => false
+      foreign_key :forked_from_id, :projects, :null => false
+      Integer :fork_id, :null => false, :unique => true
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => '0'
+      primary_key([:forked_project_id, :forked_from_id])
+    end
+  end
+end

data/lib/ghtorrent/migrations/018_drop_merged_user_from_pull_requests.rb

@@ -0,0 +1,23 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+
+  up do
+    puts 'Dropping columns merged and user_id from pull_requests'
+    alter_table :pull_requests do
+      drop_column :merged
+      drop_foreign_key :user_id
+    end
+  end
+
+  down do
+    puts 'Adding columns merged and user_id to pull_requests'
+    add_column :pull_requests, :merged, TrueClass, :null => false,
+      :default => false
+
+    add_foreign_key :user_id, :users, :null => false
+
+  end
+end

data/lib/ghtorrent/migrations/019_add_fake_to_users.rb

@@ -0,0 +1,33 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+
+Sequel.migration do
+
+  up do
+    puts 'Add column fake to users'
+      add_column :users, :fake, TrueClass, :null => false, :default => false
+
+    if self.database_type == :mysql
+      self.transaction(:rollback => :reraise, :isolation => :committed) do
+        self << "update users
+                 set fake = '1'
+                 where CAST(users.login AS BINARY) regexp '[A-Z]{8}'
+                  and not exists (select * from pull_request_history where users.id = actor_id)
+                  and not exists (select * from issue_events where actor_id = users.id)
+                  and not exists (select * from project_members where users.id = user_id)
+                  and not exists (select * from issues where reporter_id=users.id )
+                  and not exists (select * from issues where assignee_id=users.id )
+                a nd not exists (select * from organization_members where user_id = users.id);"
+      end
+    end
+  end
+
+  down do
+    puts 'Drop column fake from users'
+    alter_table :users do
+      drop_column :fake
+    end
+  end
+end

data/lib/ghtorrent/multiprocess_queue_client.rb

@@ -0,0 +1,105 @@
+class MultiprocessQueueClient < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+
+  def clazz
+    raise('Unimplemented')
+  end
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Retrieve data for multiple repos in parallel. To work, it requires
+a mapping file formatted as either of the follow formats:
+
+U IP UNAME PASSWD NUM_PROCS
+T IP TOKEN NUM_PROCS
+
+{U,T}: U signifies that a username/password pair is provided, T that an OAuth
+       token is specified instead
+IP: address to use for outgoing requests (use 0.0.0.0 on non-multihomed hosts)
+UNAME: Github user name to use for outgoing requests
+PASSWD: Github password to use for outgoing requests
+TOKEN: Github OAuth token
+NUM_PROCS: Number of processes to spawn for this IP/UNAME combination
+
+Values in the config.yaml file set with the -c command are overridden.
+
+#{command_name} [options] mapping-file
+
+    BANNER
+    options.opt :queue, 'Queue to retrieve project names from',
+                :short => 'q', :default => 'multiprocess-queue-client',
+                :type => :string
+  end
+
+  def logger
+    @logger ||= Logger.new(STDOUT)
+    @logger
+  end
+
+  def validate
+    super
+    Trollop::die 'Argument mapping-file is required' unless not args[0].nil?
+  end
+
+  def go
+
+    configs = File.open(ARGV[0]).readlines.map do |line|
+      next if line =~ /^#/
+      case line.strip.split(/ /)[0]
+        when 'U'
+          type, ip, name, passwd, instances = line.strip.split(/ /)
+        when 'T'
+          type, ip, token, instances = line.strip.split(/ /)
+      end
+
+      (1..instances.to_i).map do |i|
+        newcfg = self.settings.clone
+        newcfg = override_config(newcfg, :attach_ip, ip)
+
+        case type
+          when 'U'
+            newcfg = override_config(newcfg, :github_username, name)
+            newcfg = override_config(newcfg, :github_passwd, passwd)
+          when 'T'
+            newcfg = override_config(newcfg, :github_token, token)
+        end
+
+        newcfg = override_config(newcfg, :mirror_history_pages_back, 100000)
+        newcfg
+      end
+    end.flatten.select { |x| !x.nil? }
+
+    children = configs.map do |config|
+      pid = Process::fork
+
+      if pid.nil?
+        retriever = clazz.new(config, options[:queue])
+
+        Signal.trap('TERM') {
+          retriever.stop
+        }
+
+        retriever.run(self)
+        exit
+      else
+        debug "Parent #{Process.pid} forked child #{pid}"
+        pid
+      end
+    end
+
+    debug 'Waiting for children'
+    begin
+      children.each do |pid|
+        debug "Waiting for child #{pid}"
+        Process.waitpid(pid, 0)
+        debug "Child #{pid} exited"
+      end
+    rescue Interrupt
+      debug 'Stopping'
+    end
+  end
+end
+
+# vim: ft=ruby:

data/lib/ghtorrent/persister.rb

@@ -19,5 +19,8 @@ module GHTorrent
       driver.new(settings)
     end
 
+    def disconnect
+      driver.close
+    end
   end
-end
+end

data/lib/ghtorrent/retriever.rb

@@ -145,20 +145,22 @@ module GHTorrent
       end
     end
 
-    # Retrieve up to 30 * +:mirror_commit_pages_new_repo+ commits
-    # starting from the provided +sha+
-    def retrieve_commits(repo, sha, user, num_pages = config(:mirror_commit_pages_new_repo))
-      last_sha = if sha == "head" then "master" else sha end
+    # Retrieve commits starting from the provided +sha+
+    def retrieve_commits(repo, sha, user, pages = -1)
 
-      url = ghurl "repos/#{user}/#{repo}/commits?sha=#{last_sha}"
-      commits = paged_api_request(url, num_pages)
+      url = if sha.nil?
+              ghurl "repos/#{user}/#{repo}/commits"
+            else
+              ghurl "repos/#{user}/#{repo}/commits?sha=#{sha}"
+            end
+
+      commits = restricted_page_request(url, pages)
 
       commits.map do |c|
         retrieve_commit(repo, c['sha'], user)
       end
     end
 
-
     def retrieve_repo(user, repo)
       stored_repo = persister.find(:repos, {'owner.login' => user,
                                              'name' => repo })
@@ -253,15 +255,15 @@ module GHTorrent
     # Retrieve all collaborators for a repository
     def retrieve_repo_collaborators(user, repo)
       repo_bound_items(user, repo, :repo_collaborators,
-                       "repos/#{user}/#{repo}/collaborators",
+                       ["repos/#{user}/#{repo}/collaborators"],
                        {'repo' => repo, 'owner' => user},
-                       'login')
+                       'login', item = nil, refresh = false, order = :asc)
     end
 
     # Retrieve a single repository collaborator
     def retrieve_repo_collaborator(user, repo, new_member)
       repo_bound_item(user, repo, new_member, :repo_collaborators,
-                      "repos/#{user}/#{repo}/collaborators",
+                      ["repos/#{user}/#{repo}/collaborators"],
                       {'repo' => repo, 'owner' => user},
                       'login')
     end
@@ -269,17 +271,17 @@ module GHTorrent
     # Retrieve all watchers for a repository
     def retrieve_watchers(user, repo)
       repo_bound_items(user, repo, :watchers,
-                       "repos/#{user}/#{repo}/stargazers",
+                       ["repos/#{user}/#{repo}/stargazers"],
                        {'repo' => repo, 'owner' => user},
-                       'login')
+                       'login', item = nil, refresh = false, order = :desc)
     end
 
-    # Retrieve a single watcher for a repositry
+    # Retrieve a single watcher for a repository
     def retrieve_watcher(user, repo, watcher)
       repo_bound_item(user, repo, watcher, :watchers,
-                      "repos/#{user}/#{repo}/stargazers",
+                      ["repos/#{user}/#{repo}/stargazers"],
                       {'repo' => repo, 'owner' => user},
-                      'login')
+                      'login', order = :desc)
     end
 
     def retrieve_pull_requests(user, repo, refr = false)
@@ -288,7 +290,7 @@ module GHTorrent
       repo_bound_items(user, repo, :pull_requests,
                        [open, closed],
                        {'repo' => repo, 'owner' => user},
-                       'number', item = nil, refresh = refr)
+                       'number', item = nil, refresh = refr, order = :asc)
     end
 
     def retrieve_pull_request(user, repo, pullreq_id)
@@ -303,51 +305,27 @@ module GHTorrent
 
     def retrieve_forks(user, repo)
       repo_bound_items(user, repo, :forks,
-                       "repos/#{user}/#{repo}/forks",
+                       ["repos/#{user}/#{repo}/forks"],
                        {'repo' => repo, 'owner' => user},
-                       'id')
+                       'id', item = nil, refresh = false, order = :asc)
     end
 
     def retrieve_fork(user, repo, fork_id)
       repo_bound_item(user, repo, fork_id, :forks,
-                       "repos/#{user}/#{repo}/forks",
+                       ["repos/#{user}/#{repo}/forks"],
                        {'repo' => repo, 'owner' => user},
                        'id')
     end
 
     def retrieve_pull_req_commits(user, repo, pullreq_id)
-      def is_intra_branch(req)
-        return false if req['head'].nil? or req['head']['repo'].nil?
-        req['head']['repo']['owner']['login'] ==
-            req['base']['repo']['owner']['login'] and
-            req['head']['repo']['full_name'] == req['base']['repo']['full_name']
-      end
-
-      pull_req = retrieve_pull_request(user, repo, pullreq_id)
+      pr_commits = paged_api_request(ghurl "repos/#{user}/#{repo}/pulls/#{pullreq_id}/commits")
 
-      unless is_intra_branch(pull_req)
-
-        # Head repo has been deleted
-        unless pull_req['head']['repo'].nil?
-          head_user = pull_req['head']['repo']['owner']['login']
-          head_repo = pull_req['head']['repo']['name']
-        else
-          # Try to find the commits in the base repo, in case the pull req
-          # has been merged
-          head_user = pull_req['base']['repo']['owner']['login']
-          head_repo = pull_req['base']['repo']['name']
-        end
+      pr_commits.map do |x|
+        head_user = x['url'].split(/\//)[4]
+        head_repo = x['url'].split(/\//)[5]
 
-        commits = paged_api_request(ghurl "repos/#{user}/#{repo}/pulls/#{pullreq_id}/commits")
-        commits.map { |x|
-          retrieve_commit(head_repo, x['sha'], head_user)
-        }
-      else
-        commits = paged_api_request(ghurl "repos/#{user}/#{repo}/pulls/#{pullreq_id}/commits")
-        commits.map { |x|
-          retrieve_commit(repo, x['sha'], user)
-        }
-      end
+        retrieve_commit(head_repo, x['sha'], head_user)
+      end.select{|x| not x.nil?}
     end
 
     def retrieve_pull_req_comments(owner, repo, pullreq_id)
@@ -406,7 +384,7 @@ module GHTorrent
       repo_bound_items(user, repo, :issues,
                        [open, closed],
                        {'repo' => repo, 'owner' => user},
-                       'number', item = nil, refresh = refr)
+                       'number', item = nil, refresh = refr, order = :asc)
     end
 
     def retrieve_issue(user, repo, issue_id)
@@ -519,20 +497,16 @@ module GHTorrent
       end
     end
 
-    def retrieve_issue_labels(owner, repo, issue_id)
-
-    end
-
     def retrieve_repo_labels(owner, repo, refr = false)
       repo_bound_items(owner, repo, :repo_labels,
-                       "repos/#{owner}/#{repo}/labels",
+                       ["repos/#{owner}/#{repo}/labels"],
                        {'repo' => repo, 'owner' => owner},
-                       'name', item = nil, refresh = refr)
+                       'name', item = nil, refresh = refr, order = :asc)
     end
 
     def retrieve_repo_label(owner, repo, name)
       repo_bound_item(owner, repo, name, :repo_labels,
-                       "repos/#{owner}/#{repo}/labels",
+                       ["repos/#{owner}/#{repo}/labels"],
                        {'repo' => repo, 'owner' => owner},
                        'name')
     end
@@ -569,74 +543,96 @@ module GHTorrent
 
     private
 
-    def repo_bound_items(user, repo, entity, urls, selector, descriminator,
-                         item_id = nil, refresh = false)
+    def restricted_page_request(url, pages)
+      if pages != -1
+        paged_api_request(url, pages)
+      else
+        paged_api_request(url)
+      end
+    end
+
+    def repo_bound_items(user, repo, entity, urls, selector, discriminator,
+        item_id = nil, refresh = false, order = :asc)
 
-      items = if urls.class == Array
-                urls.map { |url| paged_api_request(ghurl url) }.flatten
-              else
-                paged_api_request(ghurl urls)
-              end
+       urls.each do |url|
+        total_pages = num_pages(ghurl url)
 
-      items = items.map do |x|
-        x['repo'] = repo
-        x['owner'] = user
+        page_range = if order == :asc
+                       (1..total_pages)
+                     else
+                       total_pages.downto(1)
+                     end
 
-        instances = repo_bound_instance(entity, selector,
-                                        descriminator, x[descriminator])
-        exists = !instances.empty?
+        page_range.each do |page|
+          items = api_request(ghurl(url, page))
 
-        unless exists
-          persister.store(entity, x)
-          info "Retriever: Added #{entity} #{user}/#{repo} -> #{x[descriminator]}"
-        else
-          if refresh
-            instances.each do |i|
+          items.each do |x|
+            x['repo'] = repo
+            x['owner'] = user
 
-              id = if i[descriminator].to_i.to_s != i[descriminator]
-                i[descriminator] # item_id is int
+            instances = repo_bound_instance(entity, selector,
+                                            discriminator, x[discriminator])
+            exists = !instances.empty?
+
+            unless exists
+              persister.store(entity, x)
+              info "Retriever: Added #{entity} #{user}/#{repo} -> #{x[discriminator]}"
+            else
+              if refresh
+                instances.each do |i|
+
+                  id = if i[discriminator].to_i.to_s != i[discriminator]
+                         i[discriminator] # item_id is int
+                       else
+                         i[discriminator].to_i # convert to int
+                       end
+
+                  instance_selector = selector.merge({discriminator => id})
+                  persister.del(entity, instance_selector)
+                  persister.store(entity, x)
+                  debug "Retriever: Refreshing #{entity} #{user}/#{repo} -> #{x[discriminator]}"
+                end
               else
-                i[descriminator].to_i # convert to int
+                debug "Retriever: #{entity} #{user}/#{repo} -> #{x[discriminator]} exists"
               end
+            end
 
-              instance_selector = selector.merge({descriminator => id})
-              persister.del(entity, instance_selector)
-              persister.store(entity, x)
-              debug "Retriever: Refreshing #{entity} #{user}/#{repo} -> #{x[descriminator]}"
+            # If we are just looking for a single item, give the method a chance
+            # to return as soon as we find it. This is to avoid loading all
+            # items before we actually search for what we are looking for.
+            unless item_id.nil?
+              a = repo_bound_instance(entity, selector, discriminator, item_id)
+              unless a.empty?
+                return a
+              end
             end
-          else
-            debug "Retriever: #{entity} #{user}/#{repo} -> #{x[descriminator]} exists"
           end
         end
-        # If the persistence driver does not set an ext_ref_id key, set a dummy
-        # one here
-        unless x.has_key? ext_uniq
-          x[ext_uniq] = '0'
-        end
-        x
       end
 
       if item_id.nil?
-        a = persister.find(entity, selector)
-        if a.empty? then items else a end
+        persister.find(entity, selector)
       else
-        a = repo_bound_instance(entity, selector, descriminator, item_id)
-        if a.empty? then [items.find{|x| x[descriminator] == item_id}] else a end
+        # If the item we are looking for has been found, the method should
+        # have returned earlier. So just return an empty result to indicate
+        # that the item has not been found.
+        []
       end
     end
 
-    def repo_bound_item(user, repo, item_id, entity, url, selector, descriminator)
-      stored_item = repo_bound_instance(entity, selector, descriminator, item_id)
+    def repo_bound_item(user, repo, item_id, entity, url, selector,
+        discriminator, order = :asc)
+      stored_item = repo_bound_instance(entity, selector, discriminator, item_id)
 
       if stored_item.empty?
-        repo_bound_items(user, repo, entity, url, selector, descriminator,
-                         item_id).first
+        repo_bound_items(user, repo, entity, url, selector, discriminator,
+                         item_id, false, order).first
       else
         stored_item.first
       end
     end
 
-    def repo_bound_instance(entity, selector, descriminator, item_id)
+    def repo_bound_instance(entity, selector, discriminator, item_id)
 
       id = if item_id.to_i.to_s != item_id
              item_id # item_id is int
@@ -644,21 +640,35 @@ module GHTorrent
              item_id.to_i # convert to int
            end
 
-      instance_selector = selector.merge({descriminator => id})
+      instance_selector = selector.merge({discriminator => id})
       result = persister.find(entity, instance_selector)
       if result.empty?
-        # Try without type conversions. Useful when the descriminator type
+        # Try without type conversions. Useful when the discriminator type
         # is string and an item_id that can be converted to int is passed.
         # Having no types sucks occasionaly...
-        instance_selector = selector.merge({descriminator => item_id})
+        instance_selector = selector.merge({discriminator => item_id})
         persister.find(entity, instance_selector)
       else
         result
       end
     end
 
-    def ghurl(path)
-      config(:mirror_urlbase) + path
+    def ghurl(path, page = -1, per_page = 100)
+      if page > 0
+        if path.include?('?')
+          path = path + "&page=#{page}&per_page=#{per_page}"
+        else
+          path = path + "?page=#{page}&per_page=#{per_page}"
+        end
+        config(:mirror_urlbase) + path
+      else
+        if path.include?('?')
+          path = path + "&per_page=#{per_page}"
+        else
+          path = path + "?per_page=#{per_page}"
+        end
+        config(:mirror_urlbase) + path
+      end
     end
 
   end