ghtorrent 0.9 → 0.10

data/lib/ghtorrent/command.rb

@@ -54,6 +54,12 @@ module GHTorrent
                                                      command.options[:password])
         end
 
+        unless command.options[:token].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :github_token,
+                                                     command.options[:token])
+        end
+
         begin
           command.go
         rescue => e
@@ -86,6 +92,8 @@ Standard options:
             :type => String
         opt :username, 'Username at Github', :short => 's', :type => String
         opt :password, 'Password at Github', :type => String
+        opt :token, 'OAuth Github token (use instead of username/password)',
+            :type => String, :short => 't'
       end
     end
 
@@ -166,13 +174,13 @@ Standard options:
                       :ack => true) do |delivery_info, properties, msg|
 
             if ack == :before
-              ch.acknowledge(delivery_info.delivery_tag, false)
+              ch.acknowledge(delivery_info.delivery_tag)
             end
 
             begin
               block.call(msg)
             ensure
-              ch.acknowledge(delivery_info.delivery_tag, false)
+              ch.acknowledge(delivery_info.delivery_tag)
             end
           end
 
@@ -197,8 +205,8 @@ Standard options:
     end
 
     def override_config(config_file, setting, new_value)
-      puts "Overriding configuration #{setting}=#{config(setting)} with cmd line #{new_value}"
-      merge_config_values(config_file, {setting => new_value})
+      puts "Overriding configuration #{setting}=#{config(setting)} with new value #{new_value}"
+      super(config_file, setting, new_value)
     end
 
     private

data/lib/ghtorrent/commands/ght_data_retrieval.rb

@@ -81,7 +81,7 @@ class GHTDataRetrieval < GHTorrent::Command
     repo = data['repo']['name'].split(/\//)[1]
     fork_id = data['payload']['forkee']['id']
 
-    ghtorrent.get_fork(owner, repo, fork_id)
+    #ghtorrent.get_fork(owner, repo, fork_id)
   end
 
   def PullRequestReviewCommentEvent(data)
@@ -119,20 +119,15 @@ class GHTDataRetrieval < GHTorrent::Command
 
   def prepare_options(options)
     options.banner <<-BANNER
-Retrieves events from queues and processes them through GHTorrent
-#{command_name} [options]
-
-#{command_name} options:
+Retrieves events from queues and processes them through GHTorrent.
+If event_id is provided, only this event is processed.
+#{command_name} [event_id]
     BANNER
 
-    options.opt :filter,
-                'Only process messages for repos in the provided file',
-                :short => 'f', :type => String
   end
 
   def validate
     super
-    Trollop::die "Filter file does not exist" if options[:filter] and not File.exist?(options[:filter])
   end
 
   def logger
@@ -144,19 +139,26 @@ Retrieves events from queues and processes them through GHTorrent
     @gh
   end
 
+  def retrieve_event(evt_id)
+    event = persister.get_underlying_connection[:events].find_one('id' => evt_id)
+    event.delete '_id'
+    data = parse(event.to_json)
+    info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
+    data
+  end
+
   def go
-    filter = Array.new
-
-    if options[:filter]
-      File.open(options[:filter]).each { |l|
-        next if l.match(/^ *#/)
-        parts = l.split(/ /)
-        next if parts.size < 2
-        debug "GHTDataRetrieval: Filtering events by #{parts[0] + "/" + parts[1]}"
-        filter << parts[0] + "/" + parts[1]
-      }
-    end
 
+    unless ARGV[0].nil?
+      event = retrieve_event(ARGV[0])
+
+      if event.nil?
+        warn "GHTDataRetrieval: No event with id: #{ARGV[0]}"
+      else
+        send(event['type'], event)
+      end
+      return
+    end
 
     conn = Bunny.new(:host => config(:amqp_host),
                      :port => config(:amqp_port),
@@ -181,20 +183,9 @@ Retrieves events from queues and processes them through GHTorrent
       queue.subscribe(:ack => true) do |headers, properties, msg|
         begin
 
-          event = persister.get_underlying_connection[:events].find_one('id' => msg)
-          event.delete '_id'
-          data = parse(event.to_json)
-          info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
-
-          unless options[:filter].nil?
-            if filter.include?(data['repo']['name'])
-              send(h, data)
-            else
-              info "GHTDataRetrieval: Repo #{data['repo']['name']} not in process list. Ignoring event #{data['type']}-#{data['id']}"
-            end
-          else
-            send(h, data)
-          end
+          data = retrieve_event(msg)
+          send(h, data)
+
           channel.acknowledge(headers.delivery_tag, false)
           info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
         rescue Exception => e
@@ -227,6 +218,7 @@ Retrieves events from queues and processes them through GHTorrent
     conn.close unless conn.nil?
 
   end
+
 end
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/commands/ght_get_more_commits.rb

@@ -61,7 +61,7 @@ Retrieves more commits for the provided repository
 
     user = user_entry[:login]
 
-    repo_entry = @ght.transaction{@ght.ensure_repo(ARGV[0], ARGV[1], false, false, false)}
+    repo_entry = @ght.transaction{@ght.ensure_repo(ARGV[0], ARGV[1])}
 
     if repo_entry.nil?
       Trollop::die "Cannot find repository #{owner}/#{ARGV[1]}"
@@ -75,7 +75,7 @@ Retrieves more commits for the provided repository
                       order(:created_at).\
                       first[:sha]
            else
-             "master"
+             nil
            end
 
     total_commits = 0
@@ -84,7 +84,8 @@ Retrieves more commits for the provided repository
       begin
         logger.debug("Retrieving more commits for #{user}/#{repo} from head: #{head}")
 
-        commits = retrieve_commits(repo, head, user, 1)
+        @settings = override_config(@settings, :mirror_history_pages_back, 1)
+        commits = retrieve_commits(repo, head, user)
 
         if commits.nil? or commits.empty? or commits.size == 1
           break

data/lib/ghtorrent/commands/ght_load.rb

@@ -107,13 +107,14 @@ Loads object ids from a collection to a queue for further processing.
                            :routing_key => "evt.#{e['type']}"
 
           total_read += 1
-          puts "Publish id = #{e['id']} (#{total_read} read)" if options.verbose
+          puts "Publish id = #{e['id']} #{e['created_at']} (#{total_read} read)" if options.verbose
 
           if total_read >= options[:number]
             puts 'Finished reading, exiting'
-            break
+            return 
           end
         end
+        stopped = true
       rescue Interrupt
         puts 'Interrupted'
         stopped = true

data/lib/ghtorrent/commands/ght_retrieve_one.rb

@@ -0,0 +1,80 @@
+require 'rubygems'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/retriever'
+
+class GHTRetrieveOne < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Retriever
+  include GHTorrent::Persister
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Retrieve just one item
+
+#{command_name} [options] <what> options...
+  what can have the following values and arguments
+    * pullreq <owner> <repo> <github_id>
+    * issue <owner> <repo> <github_id>
+    BANNER
+  end
+
+
+  def validate
+    super
+    Trollop::die 'One argument required' unless args[0] && !args[0].empty?
+  end
+
+  def logger
+    ght.logger
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def ext_uniq
+    @ext_uniq ||= config(:uniq_id)
+    @ext_uniq
+  end
+
+  def ght
+    @ght ||= TransactedGhtorrent.new(settings)
+    @ght
+  end
+
+  def go
+
+    ght.get_db
+    case ARGV[0]
+      when /pullreq/
+        retrieve_pullreq(ARGV[1..-1])
+      when /issue/
+        retrieve_issue(ARGV[1..-1])
+      else
+        Trollop::die "Don't know how to retrieve #{ARGV[0]}"
+    end
+  end
+
+  def retrieve_pullreq(args)
+    owner = args[0]
+    repo = args[1]
+    pull_req_id = args[2]
+
+    ght.ensure_pull_request(owner, repo, pull_req_id)
+  end
+
+  def retrieve_issue(args)
+    owner = args[0]
+    repo = args[1]
+    issue_id = args[2]
+
+    ght.ensure_issue(wner, repo, issue_id)
+  end
+
+end

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -41,12 +41,12 @@ An efficient way to get all data for a single repo
   end
 
   def ght
-    @ght ||= TransactedGHTorrent.new(settings)
+    @ght ||= TransactedGhtorrent.new(settings)
     @ght
   end
 
   def go
-    self.settings = override_config(settings, :mirror_history_pages_back, -1)
+    self.settings = override_config(settings, :mirror_history_pages_back, 1000)
     user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
 
     if user_entry.nil?
@@ -64,7 +64,7 @@ An efficient way to get all data for a single repo
     repo = repo_entry[:name]
 
     def send_message(function, user, repo)
-      ght.send(function, user, repo, refresh = true)
+      ght.send(function, user, repo)
     end
 
     functions = %w(ensure_commits ensure_forks ensure_pull_requests

data/lib/ghtorrent/commands/ght_retrieve_repos.rb

@@ -3,98 +3,21 @@ require 'ghtorrent/settings'
 require 'ghtorrent/logging'
 require 'ghtorrent/command'
 require 'ghtorrent/retriever'
+require 'ghtorrent/multiprocess_queue_client'
 require "bunny"
 
+class GHTRetrieveRepos < MultiprocessQueueClient
 
-class GHTRetrieveRepos < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Logging
-
-  def logger
-    @logger ||= Logger.new(STDOUT)
-    @logger
-  end
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Retrieve data for multiple repos in parallel. To work, it requires
-a mapping file formatted as follows:
-
-IP UNAME PASSWD NUM_PROCS where
-
-IP = address to use for outgoing requests (use 0.0.0.0 on non-multihomed hosts)
-UNAME = Github user name to use for outgoing requests
-PASSWD = Github password to use for outgoing requests
-NUM_PROCS = Number of processes to spawn for this IP/UNAME combination
-
-Values in the config.yaml file set with the -c command are overriden.
-
-#{command_name} [options] mapping-file
-
-    BANNER
-    options.opt :queue, 'Queue to retrieve project names from',
-                :short => 'q', :default => 'retrieve-repo', :type => :string
-
+  def clazz
+    GHTRepoRetriever
   end
 
-  def validate
-    super
-    Trollop::die 'Argument mapping-file is required' unless not args[0].nil?
-  end
-
-  def go
-
-    configs = File.open(ARGV[0]).readlines.map do |line|
-      next if line =~ /^#/
-      ip,name,passwd,instances = line.strip.split(/ /)
-      (1..instances.to_i).map do |i|
-        newcfg = self.settings.clone
-        newcfg = override_config(newcfg, :attach_ip, ip)
-        newcfg = override_config(newcfg, :github_username, name)
-        newcfg = override_config(newcfg, :github_passwd, passwd)
-        newcfg = override_config(newcfg, :mirror_history_pages_back, 1000)
-        newcfg = override_config(newcfg, :mirror_commit_pages_new_repo, 1000)
-        newcfg
-      end
-    end.flatten.select{|x| !x.nil?}
-
-    children = configs.map do |config|
-      pid = Process::fork
-
-      if pid.nil?
-        retriever = GHTRepoRetriever.new(config, options[:queue])
-
-        Signal.trap('TERM') {
-          retriever.stop
-        }
-
-        retriever.run(self)
-        exit
-      else
-        debug "Parent #{Process.pid} forked child #{pid}"
-        pid
-      end
-    end
-
-    debug 'Waiting for children'
-    begin
-      children.each do |pid|
-        debug "Waiting for child #{pid}"
-        Process.waitpid(pid, 0)
-        debug "Child #{pid} exited"
-      end
-    rescue Interrupt
-      debug 'Stopping'
-    end
-  end
 end
 
 class GHTRepoRetriever
 
   include GHTorrent::Settings
   include GHTorrent::Retriever
-  include GHTorrent::Persister
 
   def initialize(config, queue)
     @config = config
@@ -105,11 +28,6 @@ class GHTRepoRetriever
     ght.logger
   end
 
-  def persister
-    @persister ||= connect(:mongo, settings)
-    @persister
-  end
-
   def ext_uniq
     @ext_uniq ||= config(:uniq_id)
     @ext_uniq
@@ -128,14 +46,34 @@ class GHTRepoRetriever
 
     processor = Proc.new do |msg|
       owner, repo = msg.split(/ /)
-      user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
+
+      # On rare occasions, 2 instances might try to add the same user
+      # at the same time, which might lead to transaction conflicts
+      # Give the script one more opportunity before bailing out
+      user_entry = nil
+      i = 0
+
+      while user_entry.nil? and i < 10 do
+        i += 1
+        warn("Trying to get user #{owner}, attempt #{i}")
+        begin
+          user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
+        rescue Exception => e
+          warn e.message
+        end
+      end
 
       if user_entry.nil?
         warn("Cannot find user #{owner}")
         next
       end
 
-      repo_entry = ght.transaction { ght.ensure_repo(owner, repo) }
+      repo_entry = ght.transaction { ght.ensure_repo(owner, repo,
+                                                     commits = false,
+                                                     project_members = false,
+                                                     watchers = false,
+                                                     forks = false,
+                                                     labels = false) }
 
       if repo_entry.nil?
         warn("Cannot find repository #{owner}/#{repo}")
@@ -144,30 +82,49 @@ class GHTRepoRetriever
 
       debug("Retrieving repo #{owner}/#{repo}")
 
-      def send_message(function, user, repo)
-        ght.send(function, user, repo, refresh = false)
-      end
+      retrieval_stages = %w(ensure_commits ensure_forks ensure_pull_requests
+                            ensure_issues ensure_project_members
+                            ensure_watchers ensure_labels)
 
-      functions = %w(ensure_commits ensure_forks ensure_pull_requests
-            ensure_issues ensure_project_members ensure_watchers ensure_labels)
+      retrieval_stages.each do |x|
+        run_retrieval_stage(ght, owner, repo, x)
+      end
 
-      functions.each do |x|
+      # Repository owner bound data retrieval
+      run_retrieval_stage(ght, owner, repo, 'ensure_user_followers', 
+                          onlyuser = true)
 
-        begin
-          send_message(x, owner, repo)
-        rescue Exception
-          warn("Error processing #{x} for #{owner}/#{repo}")
-          next
-        end
+      if user_entry[:type] == 'ORG'
+        run_retrieval_stage(ght, owner, repo, 'ensure_org', onlyuser = true)
       end
+
+      # Cleanup
+      ght.dispose
+      ght = nil
+      GC.start
     end
 
     command.queue_client(@queue, :before, processor)
   end
 
+  def run_retrieval_stage(ght, owner, repo, function, only_user = false)
+    begin
+      if only_user
+        ght.send(function, owner)
+      else
+        ght.send(function, owner, repo)
+      end
+    rescue Exception => e
+      warn("Error processing #{function} for #{owner}/#{repo}")
+      warn("Exception message #{$!}")
+      warn("Exception trace #{e.backtrace.join("\n")}")
+    end
+  end
+
   def stop
     warn('Stop flag set, waiting for operations to finish')
     @stop = true
   end
 end
 
+# vim: ft=ruby: