ghtorrent 0.10 → 0.11

data/lib/ghtorrent/command.rb

@@ -17,6 +17,7 @@ module GHTorrent
   class Command
 
     include GHTorrent::Settings
+    include GHTorrent::Logging
 
     # Specify the run method for subclasses.
     class << self
@@ -60,6 +61,19 @@ module GHTorrent
                                                      command.options[:token])
         end
 
+        unless command.options[:req_limit].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :req_limit,
+                                                     command.options[:req_limit])
+        end
+
+        unless command.options[:uniq].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :logging_uniq,
+                                                     command.options[:uniq])
+        end
+
+
         begin
           command.go
         rescue => e
@@ -88,12 +102,16 @@ Standard options:
         opt :config, 'config.yaml file location', :short => 'c',
             :default => 'config.yaml'
         opt :verbose, 'verbose mode', :short => 'v'
-        opt :addr, 'ip address to use for performing requests', :short => 'a',
+        opt :addr, 'IP address to use for performing requests', :short => 'a',
             :type => String
         opt :username, 'Username at Github', :short => 's', :type => String
         opt :password, 'Password at Github', :type => String
         opt :token, 'OAuth Github token (use instead of username/password)',
             :type => String, :short => 't'
+        opt :req_limit, 'Request limit for provided account (in reqs/hour)',
+            :type => Integer, :short => 'l'
+        opt :uniq, 'Unique name for this command. Will appear in logs.',
+            :type => String, :short => 'u'
       end
     end
 
@@ -143,12 +161,11 @@ Standard options:
     def go
     end
 
-    # Specify a handler to incoming messages from a connection to
-    # a queue.
-    # [queue]: The queue name to bind to
-    # [ack]: :before or :after when should acks be send, before or after
-    #        the block returns
-    # [block]: A block with one argument (the message)
+    # Specify a handler to incoming messages from a connection to a queue.
+    #
+    # @param queue [String] the queue name to bind to
+    # @param ack [Symbol] when should acks be send, :before or :after the block returns
+    # @param block [Block]: A block accepting one argument (the message)
     def queue_client(queue, ack = :after, block)
 
       stopped = false
@@ -194,7 +211,7 @@ Standard options:
           sleep(1)
         rescue Interrupt => _
           stopped = true
-        rescue Exception => e
+        rescue StandardError => e
           raise e
         end
       end

data/lib/ghtorrent/commands/full_user_retriever.rb

@@ -0,0 +1,50 @@
+module GHTorrent
+  module Commands
+    # Defines a process to download the full data available for a single user
+    module FullUserRetriever
+
+      def retrieve_user(login)
+        #self.settings = override_config(settings, :mirror_history_pages_back, -1)
+
+        user_entry = ght.transaction { ght.ensure_user(login, false, false) }
+        on_github = api_request(ghurl ("users/#{login}"))
+
+        if on_github.empty?
+          if user_entry.nil?
+            warn "User #{login} does not exist on GitHub"
+            exit
+          else
+            ght.transaction do
+              ght.get_db.from(:users).where(:login => login).update(:users__deleted => true)
+            end
+            warn "User #{login} marked as deleted"
+            return
+          end
+        else
+          if user_entry.nil?
+            warn "Error retrieving user #{login}"
+            exit
+          end
+        end
+
+        user = user_entry[:login]
+
+        def send_message(function, user)
+          begin
+            ght.send(function, user)
+          rescue StandardError => e
+            puts STDERR, e.message
+            puts STDERR, e.backtrace
+          end
+        end
+
+        functions = %w(ensure_user_following ensure_user_followers ensure_orgs ensure_org)
+
+        functions.each do |x|
+          send_message(x, user)
+        end
+
+      end
+    end
+  end
+ end

data/lib/ghtorrent/commands/ght_data_retrieval.rb

@@ -12,6 +12,7 @@ class GHTDataRetrieval < GHTorrent::Command
   include GHTorrent::Settings
   include GHTorrent::Logging
   include GHTorrent::Persister
+  include GHTorrent::EventProcessing
 
   def persister
     @persister ||= connect(:mongo, settings)
@@ -22,99 +23,11 @@ class GHTDataRetrieval < GHTorrent::Command
     JSON.parse(msg)
   end
 
-  def PushEvent(data)
-    data['payload']['commits'].each do |c|
-      url = c['url'].split(/\//)
-
-      ghtorrent.get_commit url[4], url[5], url[7]
-    end
-  end
-
-  def WatchEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    watcher = data['actor']['login']
-    created_at = data['created_at']
-
-    ghtorrent.get_watcher owner, repo, watcher, created_at
-  end
-
-  def FollowEvent(data)
-    follower = data['actor']['login']
-    followed = data['payload']['target']['login']
-    created_at = data['created_at']
-
-    ghtorrent.get_follower(follower, followed, created_at)
-  end
-
-  def MemberEvent(data)
-    owner = data['actor']['login']
-    repo = data['repo']['name'].split(/\//)[1]
-    new_member = data['payload']['member']['login']
-    created_at = data['created_at']
-
-    ghtorrent.get_project_member(owner, repo, new_member, created_at)
-  end
-
-  def CommitCommentEvent(data)
-    user = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    id = data['payload']['comment']['id']
-    sha = data['payload']['comment']['commit_id']
-
-    ghtorrent.get_commit_comment(user, repo, sha, id)
-  end
-
-  def PullRequestEvent(data)
-    owner = data['payload']['pull_request']['base']['repo']['owner']['login']
-    repo = data['payload']['pull_request']['base']['repo']['name']
-    pullreq_id = data['payload']['number']
-    action = data['payload']['action']
-    actor = data['actor']['login']
-    created_at = data['created_at']
-
-    ghtorrent.get_pull_request(owner, repo, pullreq_id, action, actor, created_at)
-  end
-
-  def ForkEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    fork_id = data['payload']['forkee']['id']
-
-    #ghtorrent.get_fork(owner, repo, fork_id)
-  end
-
-  def PullRequestReviewCommentEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    comment_id = data['payload']['comment']['id']
-    pullreq_id = data['payload']['comment']['_links']['pull_request']['href'].split(/\//)[-1]
-
-    ghtorrent.get_pullreq_comment(owner, repo, pullreq_id, comment_id)
-  end
-
-  def IssuesEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    issue_id = data['payload']['issue']['number']
-
-    ghtorrent.get_issue(owner, repo, issue_id)
-  end
-
-  def IssueCommentEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    issue_id = data['payload']['issue']['number']
-    comment_id = data['payload']['comment']['id']
-
-    ghtorrent.get_issue_comment(owner, repo, issue_id, comment_id)
-  end
-
   def handlers
-    %w(PushEvent WatchEvent FollowEvent MemberEvent
+    %w(PushEvent WatchEvent FollowEvent MemberEvent CreateEvent
         CommitCommentEvent PullRequestEvent ForkEvent
         PullRequestReviewCommentEvent IssuesEvent IssueCommentEvent)
-    #%w(PullRequestEvent)
+    #%w(ForkEvent)
   end
 
   def prepare_options(options)
@@ -135,7 +48,8 @@ If event_id is provided, only this event is processed.
   end
 
   def ghtorrent
-    @gh ||= GHTorrent::Mirror.new(@settings)
+    #@gh ||= GHTorrent::Mirror.new(@settings)
+    @gh ||= TransactedGHTorrent.new(settings)
     @gh
   end
 
@@ -143,7 +57,7 @@ If event_id is provided, only this event is processed.
     event = persister.get_underlying_connection[:events].find_one('id' => evt_id)
     event.delete '_id'
     data = parse(event.to_json)
-    info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
+    debug "Processing event: #{data['type']}-#{data['id']}"
     data
   end
 
@@ -153,7 +67,7 @@ If event_id is provided, only this event is processed.
       event = retrieve_event(ARGV[0])
 
       if event.nil?
-        warn "GHTDataRetrieval: No event with id: #{ARGV[0]}"
+        warn "No event with id: #{ARGV[0]}"
       else
         send(event['type'], event)
       end
@@ -178,20 +92,20 @@ If event_id is provided, only this event is processed.
       queue = channel.queue("#{h}s", {:durable => true})\
                          .bind(exchange, :routing_key => "evt.#{h}")
 
-      info "GHTDataRetrieval: Binding handler #{h} to routing key evt.#{h}"
+      info "Binding handler #{h} to routing key evt.#{h}"
 
       queue.subscribe(:ack => true) do |headers, properties, msg|
+        start = Time.now
         begin
-
           data = retrieve_event(msg)
           send(h, data)
 
           channel.acknowledge(headers.delivery_tag, false)
-          info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
-        rescue Exception => e
+          info "Success processing event. Type: #{data['type']}, ID: #{data['id']}, Time: #{Time.now.to_ms - start.to_ms} ms"
+        rescue StandardError => e
           # Give a message a chance to be reprocessed
           if headers.redelivered?
-            warn "GHTDataRetrieval: Could not process event: #{msg}"
+            warn "Error processing event. Type: #{data['type']}, ID: #{data['id']}, Time: #{Time.now.to_ms - start.to_ms} ms"
             channel.reject(headers.delivery_tag, false)
           else
             channel.reject(headers.delivery_tag, true)

data/lib/ghtorrent/commands/ght_get_more_commits.rb

@@ -12,6 +12,7 @@ class GHTMoreCommitsRetriever < GHTorrent::Command
   include GHTorrent::Settings
   include GHTorrent::Retriever
   include GHTorrent::Persister
+  include GHTorrent::Logging
 
   def prepare_options(options)
     options.banner <<-BANNER
@@ -28,7 +29,7 @@ Retrieves more commits for the provided repository
                         If not set, will start from latest stored commit',
                 :short => 'f', :default => false, :type => :boolean
     options.opt :upto, 'Get all commits up to the provided timestamp',
-                :short => 't', :default => 0, :type => :int
+                :short => 'x', :default => 0, :type => :int
   end
 
   def validate
@@ -36,20 +37,15 @@ Retrieves more commits for the provided repository
     Trollop::die "Two arguments are required" unless args[0] && !args[0].empty?
   end
 
-  def logger
-    @ght.logger
-  end
+  #def logger
+  #  @ght.logger
+  #end
 
   def persister
     @persister ||= connect(:mongo, settings)
     @persister
   end
 
-  def ext_uniq
-    @ext_uniq ||= config(:uniq_id)
-    @ext_uniq
-  end
-
   def go
 
     @ght ||= GHTorrent::Mirror.new(settings)
@@ -82,7 +78,7 @@ Retrieves more commits for the provided repository
     old_head = nil
     while (true)
       begin
-        logger.debug("Retrieving more commits for #{user}/#{repo} from head: #{head}")
+        debug("Retrieving more commits for #{user}/#{repo} from head: #{head}")
 
         @settings = override_config(@settings, :mirror_history_pages_back, 1)
         commits = retrieve_commits(repo, head, user)
@@ -97,12 +93,12 @@ Retrieves more commits for the provided repository
           total_commits += 1
 
           if options[:num] < total_commits
-            logger.info("Already retrieved #{total_commits} commits. Stopping.")
+            info("Already retrieved #{total_commits} commits. Stopping.")
             return
           end
 
           if Time.parse(c['commit']['author']['date']) < Time.at(options[:upto])
-            logger.info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
+            info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
             return
           end
 
@@ -110,17 +106,17 @@ Retrieves more commits for the provided repository
             @ght.ensure_commit(repo, c['sha'], user)
           end
         end
-      rescue Exception => e
-        logger.warn("Error processing: #{e}")
-        logger.warn(e.backtrace.join("\n"))
+      rescue StandardError => e
+        warn("Error processing: #{e}")
+        warn(e.backtrace.join("\n"))
         if old_head == head
-          logger.info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
+          info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
           fail("Cannot retrieve commits from head: #{head}")
         end
         old_head = head
       end
     end
-    logger.debug("Processed #{total_commits} commits for #{user}/#{repo}")
+    debug("Processed #{total_commits} commits for #{user}/#{repo}")
   end
 end
 

data/lib/ghtorrent/commands/ght_load.rb

@@ -4,7 +4,6 @@ require 'pp'
 require 'bunny'
 
 require 'ghtorrent/settings'
-require 'ghtorrent/logging'
 require 'ghtorrent/persister'
 require 'ghtorrent/command'
 require 'ghtorrent/bson_orderedhash'
@@ -31,7 +30,7 @@ Loads object ids from a collection to a queue for further processing.
     options.opt :earliest, 'Seconds since epoch of earliest item to load',
                 :short => 'e', :default => 0, :type => :int
     options.opt :latest, 'Seconds since epoch of latest item to load',
-                :short => 'l', :default => Time.now.to_i + (60 * 60 * 24 * 360 * 20),
+                :short => 'x', :default => Time.now.to_i + (60 * 60 * 24 * 360 * 20),
                 :type => :int
     options.opt :number, 'Total number of items to load',
                 :short => 'n', :type => :int, :default => 2**48

data/lib/ghtorrent/commands/ght_mirror_events.rb

@@ -15,10 +15,7 @@ class GHTMirrorEvents < GHTorrent::Command
   include GHTorrent::Logging
   include GHTorrent::Persister
   include GHTorrent::APIClient
-
-  def logger
-    @logger
-  end
+  include GHTorrent::Logging
 
   def store_count(events)
     stored = Array.new
@@ -41,13 +38,13 @@ class GHTMirrorEvents < GHTorrent::Command
   def retrieve(exchange)
     begin
       new = dupl = 0
-      events = api_request "https://api.github.com/events", false
+      events = api_request "https://api.github.com/events?per_page=100"
       (new, dupl, stored) = store_count events
 
-      # This means that first page cannot contain all new events. Go
-      # up to 10 pages back to find all new events not contained in first page.
+      # This means that the first page does not contain all new events. Do
+      # a paged request and get everything on the queue
       if dupl == 0
-        events = paged_api_request "https://api.github.com/events"
+        events = paged_api_request "https://api.github.com/events?per_page=100"
         (new1, dupl1, stored1) = store_count events
         stored = stored | stored1
         new = new + new1
@@ -58,7 +55,7 @@ class GHTMirrorEvents < GHTorrent::Command
         exchange.publish e['id'], :persistent => true, :routing_key => key
       end
       return new, dupl
-    rescue Exception => e
+    rescue StandardError => e
       STDERR.puts e.message
       STDERR.puts e.backtrace
     end
@@ -66,7 +63,6 @@ class GHTMirrorEvents < GHTorrent::Command
 
   def go
     @persister = connect(:mongo, @settings)
-    @logger = Logger.new(STDOUT)
 
     conn = Bunny.new(:host => config(:amqp_host),
                      :port => config(:amqp_port),
@@ -75,7 +71,7 @@ class GHTMirrorEvents < GHTorrent::Command
     conn.start
 
     ch  = conn.create_channel
-    @logger.debug "Connection to #{config(:amqp_host)} succeded"
+    debug "Connection to #{config(:amqp_host)} succeded"
 
     exchange = ch.topic(config(:amqp_exchange), :durable => true,
                  :auto_delete => false)
@@ -97,7 +93,7 @@ class GHTMirrorEvents < GHTorrent::Command
         end
       rescue Interrupt
         stopped = true
-      rescue Exception => e
+      rescue StandardError => e
         @logger.error e
       end
     end