ghtorrent 0.7.3 → 0.8

data/CHANGELOG

@@ -1,3 +1,13 @@
+= Version 0.8
+	* Retrieve and process issue labels
+	* Retrive and process actors for pull request events
+	* Retrieve pullreq comments for projects with no issuetracker
+	* Better heuristics for avoiding dulpicate entries in pull request histories
+	* The event loader now loads events by IDs to reduce pressure to the queue
+	* Compound indexes in MongoDB by default
+	* X-RateLimit-Reset header support
+	* Remove lots of dead code, general cleanups
+
 = Version 0.7.3
 	* Support for running in standalone mode
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ghtorrent (0.7.2)
+    ghtorrent (0.7.3)
       amqp (~> 1.0.0)
       bson_ext (~> 1.8.0)
       daemons (~> 1.1.0)

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -4,7 +4,8 @@ module GHTorrent
 
     ENTITIES = [:users, :commits, :followers, :repos, :events, :org_members,
         :commit_comments, :repo_collaborators, :watchers, :pull_requests,
-        :forks, :pull_request_comments, :issue_comments, :issues, :issue_events
+        :forks, :pull_request_comments, :issue_comments, :issues, :issue_events,
+        :repo_labels
     ]
 
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -20,6 +20,25 @@ module GHTorrent
         :mongo_replicas => "mongo.replicas"
     }
 
+    IDXS = {
+        :events                => %w(id),
+        :users                 => %w(login),
+        :commits               => %w(sha),
+        :commit_comments       => %w(repo user commit_id),
+        :repos                 => %w(name owner.login),
+        :repo_labels           => %w(repo owner),
+        :repo_collaborators    => %w(repo owner login),
+        :followers             => %w(follows login),
+        :org_members           => %w(org),
+        :watchers              => %w(repo owner login),
+        :forks                 => %w(repo owner id),
+        :pull_requests         => %w(repo owner number),
+        :pull_request_comments => %w(repo owner pullreq_id id),
+        :issues                => %w(repo owner number),
+        :issue_events          => %w(repo owner issue_id id),
+        :issue_comments        => %w(repo owner issue_id id)
+    }
+
     attr_reader :settings
 
     # Creates a new instance of the MongoDB persistence adapter.
@@ -122,6 +141,8 @@ module GHTorrent
           get_collection("issue_comments")
         when :issue_events
           get_collection("issue_events")
+        when :repo_labels
+          get_collection("repo_labels")
       end
     end
 
@@ -140,68 +161,35 @@ module GHTorrent
                    Mongo::ReplSetConnection.new(repl_arr, :read => :secondary)\
                                            .db(config(:mongo_db))
                  end
-        init_db(@mongo) if @mongo.collections.size < ENTITIES.size
+
+        stats = @mongo.stats
+        init_db(@mongo) if stats['collections'] < ENTITIES.size + 2
+        init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size
+
         @mongo
       else
         @mongo
       end
     end
 
-    # Declare an index on +field+ for +collection+ if it does not exist
-    def ensure_index(collection, field)
-      col = get_entity(collection)
-
-      exists = col.index_information.find {|k,v|
-        k == "#{field}_1"
-      }
-
-      if exists.nil?
-        col.create_index(field, :background => true)
-        STDERR.puts "Creating index on #{collection}(#{field})"
-      end
-    end
-
     def init_db(mongo)
       ENTITIES.each {|x| mongo.collection(x.to_s)}
 
       # Ensure that the necessary indexes exist
-      ensure_index(:events, "id")
-      ensure_index(:users, "login")
-      ensure_index(:commits, "sha")
-      ensure_index(:repos, "name")
-      ensure_index(:repos, "owner.login")
-      ensure_index(:followers, "follows")
-      ensure_index(:followers, "login")
-      ensure_index(:org_members, "org")
-      ensure_index(:commit_comments, "repo")
-      ensure_index(:commit_comments, "user")
-      ensure_index(:commit_comments, "commit_id")
-      ensure_index(:repo_collaborators, "repo")
-      ensure_index(:repo_collaborators, "owner")
-      ensure_index(:repo_collaborators, "login")
-      ensure_index(:watchers, "repo")
-      ensure_index(:watchers, "owner")
-      ensure_index(:watchers, "login")
-      ensure_index(:pull_requests, "repo")
-      ensure_index(:pull_requests, "owner")
-      ensure_index(:forks, "repo")
-      ensure_index(:forks, "owner")
-      ensure_index(:forks, "id")
-      ensure_index(:issue_comments, "repo")
-      ensure_index(:issue_comments, "owner")
-      ensure_index(:issue_comments, "issue_id")
-      ensure_index(:issue_comments, "id")
-      ensure_index(:pull_request_comments, "repo")
-      ensure_index(:pull_request_comments, "owner")
-      ensure_index(:pull_request_comments, "pullreq_id")
-      ensure_index(:pull_request_comments, "id")
-      ensure_index(:issues, "repo")
-      ensure_index(:issues, "owner")
-      ensure_index(:issues, "issue_id")
-      ensure_index(:issue_events, "repo")
-      ensure_index(:issue_events, "owner")
-      ensure_index(:issue_events, "issue_id")
-      ensure_index(:issue_events, "id")
+      IDXS.each do |k,v|
+        col = get_entity(k)
+        name = v.join('_1_') + '_1'
+        exists = col.index_information.find {|k,v| k == name}
+
+        idx_fields = v.reduce({}){|acc, x| acc.merge({x => 1})}
+        if exists.nil?
+          col.create_index(idx_fields, :background => true)
+          STDERR.puts "Creating index on #{collection}(#{v})"
+        else
+          STDERR.puts "Index on #{collection}(#{v}) exists"
+        end
+
+      end
     end
 
     def rescue_connection_failure(max_retries=60)

data/lib/ghtorrent/api_client.rb

@@ -169,14 +169,7 @@ module GHTorrent
             end
 
         total = Time.now.to_ms - start_time.to_ms
-        debug "APIClient: Request: #{url} #{if from_cache then "from cache," else "(#{contents.meta['x-ratelimit-remaining']} remaining)," end} Total: #{total} ms"
-
-        if not from_cache and config(:respect_api_ratelimit) and
-            contents.meta['x-ratelimit-remaining'].to_i < 20
-          sleep = 61 - Time.now.min
-          debug "APIClient: Request limit reached, sleeping for #{sleep} min"
-          sleep(sleep * 60)
-        end
+        debug "APIClient: Request: #{url} #{if from_cache then "from cache," else "(#{@remaining} remaining)," end} Total: #{total} ms"
 
         contents
       rescue OpenURI::HTTPError => e
@@ -188,21 +181,29 @@ module GHTorrent
               404, # Not found
               422 then # Unprocessable entity
             warn "APIClient: #{url}: #{e.io.status[1]}"
+            @remaining = e.io.meta['x-ratelimit-remaining'].to_i
+            @reset = e.io.meta['x-ratelimit-reset'].to_i
             return nil
           else # Server error or HTTP conditions that Github does not report
             warn "APIClient: #{url}: #{e.io.status[1]}"
             raise e
         end
+      ensure
+        if not from_cache and config(:respect_api_ratelimit) and @remaining < 10
+          sleep = (@reset - Time.now.to_i) / 60
+          debug "APIClient: Request limit reached, sleeping for #{sleep} min"
+          sleep(@reset - Time.now.to_i)
+        end
       end
     end
 
     def do_request(url)
-      @attach_ip ||= config(:attach_ip)
-      @username ||= config(:github_username)
-      @passwd ||= config(:github_passwd)
+      @attach_ip  ||= config(:attach_ip)
+      @username   ||= config(:github_username)
+      @passwd     ||= config(:github_passwd)
       @user_agent ||= config(:user_agent)
 
-      @open_func ||= if @username.nil?
+      open_func ||= if @username.nil?
         lambda {|url| open(url, 'User-Agent' => @user_agent)}
       else
         lambda {|url| open(url,
@@ -210,13 +211,16 @@ module GHTorrent
                            :http_basic_authentication => [@username, @passwd])}
       end
 
-      if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0'
-        @open_func.call(url)
-      else
-        attach_to(@attach_ip) do
-          @open_func.call(url)
+      result = if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0'
+          open_func.call(url)
+        else
+          attach_to(@attach_ip) do
+            open_func.call(url)
+          end
         end
-      end
+      @remaining = result.meta['x-ratelimit-remaining'].to_i
+      @reset = result.meta['x-ratelimit-reset'].to_i
+      result
     end
 
     # Attach to a specific IP address if the machine has multiple

data/lib/ghtorrent/command.rb

@@ -17,7 +17,6 @@ module GHTorrent
   class Command
 
     include GHTorrent::Settings
-    include GHTorrent::Settings
 
     # Specify the run method for subclasses.
     class << self

data/lib/ghtorrent/commands/ght_data_retrieval.rb

@@ -12,6 +12,12 @@ class GHTDataRetrieval < GHTorrent::Command
 
   include GHTorrent::Settings
   include GHTorrent::Logging
+  include GHTorrent::Persister
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
 
   def parse(msg)
     JSON.parse(msg)
@@ -64,9 +70,10 @@ class GHTDataRetrieval < GHTorrent::Command
     repo = data['payload']['pull_request']['base']['repo']['name']
     pullreq_id = data['payload']['number']
     action = data['payload']['action']
+    actor = data['actor']['login']
     created_at = data['created_at']
 
-    ghtorrent.get_pull_request(owner, repo, pullreq_id, action, created_at)
+    ghtorrent.get_pull_request(owner, repo, pullreq_id, action, actor, created_at)
   end
 
   def ForkEvent(data)
@@ -155,6 +162,7 @@ Retrieves events from queues and processes them through GHTorrent
       info "GHTDataRetrieval: Received SIGINT, exiting"
       AMQP.stop { EM.stop }
     }
+
     Signal.trap('TERM') {
       info "GHTDataRetrieval: Received SIGTERM, exiting"
       AMQP.stop { EM.stop }
@@ -165,7 +173,8 @@ Retrieves events from queues and processes them through GHTorrent
                :username => config(:amqp_username),
                :password => config(:amqp_password)) do |connection|
 
-      channel = AMQP::Channel.new(connection, :prefetch => config(:amqp_prefetch))
+      channel = AMQP::Channel.new(connection)
+      channel.prefetch(config(:amqp_prefetch))
       exchange = channel.topic(config(:amqp_exchange), :durable => true,
                                :auto_delete => false)
 
@@ -177,7 +186,10 @@ Retrieves events from queues and processes them through GHTorrent
 
         queue.subscribe(:ack => true) do |headers, msg|
           begin
-            data = parse(msg)
+
+            event = persister.get_underlying_connection[:events].find_one('id' => msg)
+            event.delete '_id'
+            data = parse(event.to_json)
             info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
 
             unless options[:filter].nil?
@@ -194,8 +206,7 @@ Retrieves events from queues and processes them through GHTorrent
           rescue Exception => e
             # Give a message a chance to be reprocessed
             if headers.redelivered?
-              data = parse(msg)
-              warn "GHTDataRetrieval: Could not process event: #{data['type']}-#{data['id']}"
+              warn "GHTDataRetrieval: Could not process event: #{msg}"
               headers.reject(:requeue => false)
             else
               headers.reject(:requeue => true)

data/lib/ghtorrent/commands/ght_load.rb

@@ -1,10 +1,8 @@
 require 'rubygems'
 require 'mongo'
 require 'amqp'
-require 'set'
 require 'eventmachine'
 require 'pp'
-require "amqp/extensions/rabbitmq"
 
 require 'ghtorrent/settings'
 require 'ghtorrent/logging'
@@ -17,27 +15,8 @@ class GHTLoad < GHTorrent::Command
   include GHTorrent::Settings
   include GHTorrent::Persister
 
-  def col_info()
-    {
-        :commits => {
-            :name => "commits",
-            :payload => "commit.id",
-            :unq => "commit.id",
-            :col => persister.get_underlying_connection.collection(:commits.to_s),
-            :routekey => "commit.%s"
-        },
-        :events => {
-            :name => "events",
-            :payload => "",
-            :unq => "type",
-            :col => persister.get_underlying_connection.collection(:events.to_s),
-            :routekey => "evt.%s"
-        }
-    }
-  end
-
   def persister
-    @persister ||= connect(:mongo, @settings)
+    @persister ||= connect(:mongo, settings)
     @persister
   end
 
@@ -52,8 +31,13 @@ Loads object ids from a collection to a queue for further processing.
 
     options.opt :earliest, 'Seconds since epoch of earliest item to load',
                 :short => 'e', :default => 0, :type => :int
-    options.opt :number, 'Number of items to load (-1 means all)',
-                :short => 'n', :type => :int, :default => -1
+    options.opt :latest, 'Seconds since epoch of latest item to load',
+                :short => 'l', :default => Time.now.to_i + (60 * 60 * 24 * 360 * 20),
+                :type => :int
+    options.opt :number, 'Total number of items to load',
+                :short => 'n', :type => :int, :default => 2**48
+    options.opt :batch, 'Number of items to process in a batch',
+                :short => 'b', :type => :int, :default => 10000
     options.opt :filter,
                 'Filter items by regexp on item attributes: item.attr=regexp',
                 :short => 'f', :type => String, :multi => true
@@ -61,7 +45,6 @@ Loads object ids from a collection to a queue for further processing.
 
   def validate
     super
-    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
     filter = options[:filter]
     case
       when filter.is_a?(Array)
@@ -71,27 +54,23 @@ Loads object ids from a collection to a queue for further processing.
       when filter == []
         # Noop
       else
-        Trollop::die "A filter can only be a string"
+        Trollop::die 'A filter can only be a string'
     end
   end
 
   def go
-    # Message tags await publisher ack
-    awaiting_ack = SortedSet.new
-
     # Num events read
     num_read = 0
 
-    collection = case args[0]
-                   when "events"
-                     :events
-                   when "commits"
-                     :commits
-                 end
-
-    puts "Loading from collection #{collection}"
     puts "Loading items after #{Time.at(options[:earliest])}" if options[:verbose]
-    puts "Loading #{options[:number]} items" if options[:verbose] && options[:number] != -1
+    puts "Loading items before #{Time.at(options[:latest])}" if options[:verbose]
+    puts "Loading #{options[:batch]} items per batch" if options[:batch]
+    puts "Loading #{options[:number]} items" if options[:verbose]
+
+    if options[:batch] >= options[:number]
+      puts "Batch > number of items, setting batch to #{options[:number]}"
+      options[:batch] = options[:number]
+    end
 
     what = case
              when options[:filter].is_a?(Array)
@@ -104,9 +83,12 @@ Loads object ids from a collection to a queue for further processing.
                {}
            end
 
-    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
+    from = {'_id' => {
+        '$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest])),
+        '$lte' => BSON::ObjectId.from_time(Time.at(options[:latest]))}
+    }
 
-    (puts "Mongo filter:"; pp what.merge(from)) if options[:verbose]
+    (puts 'Mongo filter:'; pp what.merge(from)) if options[:verbose]
 
     AMQP.start(:host => config(:amqp_host),
                :port => config(:amqp_port),
@@ -119,94 +101,47 @@ Loads object ids from a collection to a queue for further processing.
 
       # What to do when the user hits Ctrl+c
       show_stopper = Proc.new {
+        puts('Closing connection')
         connection.close { EventMachine.stop }
       }
 
-      # Read next 100000 items and queue them
+      # Read next options[:batch] items and queue them
       read_and_publish = Proc.new {
 
-        to_read = if options.number == -1
-                    100000
-                  else
-                    if options.number - num_read - 1 <= 0
-                      -1
-                    else
-                      options.number - num_read - 1
-                    end
-                  end
-
-        read = 0
-        col_info[collection][:col].find(what.merge(from),
+        persister.get_underlying_connection[:events].find(what.merge(from),
                                         :skip => num_read,
-                                        :limit => to_read).each do |e|
-
-          payload = read_value(e, col_info[collection][:payload])
-          payload = if payload.class == BSON::OrderedHash
-                      payload.delete "_id" # Inserted by MongoDB on event insert
-                      payload.to_json
-                    end
-          read += 1
-          unq = read_value(e, col_info[collection][:unq])
+                                        :limit => options[:batch]).each do |e|
+          unq = read_value(e, 'type')
           if unq.class != String or unq.nil? then
-            throw Exception.new("Unique value can only be a String")
+            throw Exception.new('Unique value can only be a String')
           end
 
-          key = col_info[collection][:routekey] % unq
-
-          exchange.publish payload, :persistent => true, :routing_key => key
+          exchange.publish e['id'], :persistent => false,
+                           :routing_key => "evt.#{e['type']}"
 
           num_read += 1
-          puts("Publish id = #{payload[unq]} (#{num_read} total)") if options.verbose
-          awaiting_ack << num_read
+          puts "Publish id = #{e['id']} (#{num_read} total)" if options.verbose
         end
 
-        # Nothing new in the DB and no msgs waiting ack
-        if (read == 0 and awaiting_ack.size == 0) or to_read == -1
-          puts("Finished reading, exiting")
+        if num_read >= options[:number]
+          puts 'Finished reading, exiting'
           show_stopper.call
-        end
-      }
-
-      # Remove acknowledged or failed msg tags from the queue
-      # Trigger more messages to be read when ack msg queue size drops to zero
-      publisher_event = Proc.new { |ack|
-        if ack.multiple then
-          awaiting_ack.delete_if { |x| x <= ack.delivery_tag }
         else
-          awaiting_ack.delete ack.delivery_tag
-        end
-
-        if awaiting_ack.size == 0
-          puts("ACKS.size= #{awaiting_ack.size}") if options.verbose
+          # Schedule new event processing cycle
           EventMachine.next_tick do
             read_and_publish.call
           end
         end
       }
 
-      # Await publisher confirms
-      channel.confirm_select
-
-      # Callback when confirms have arrived
-      channel.on_ack do |ack|
-        puts "ACK: tag=#{ack.delivery_tag}, mul=#{ack.multiple}" if options.verbose
-        publisher_event.call(ack)
-      end
-
-      # Callback when confirms failed.
-      channel.on_nack do |nack|
-        puts "NACK: tag=#{nack.delivery_tag}, mul=#{nack.multiple}" if options.verbose
-        publisher_event.call(nack)
-      end
-
       # Signal handlers
       Signal.trap('INT', show_stopper)
       Signal.trap('TERM', show_stopper)
 
-      # Trigger start processing
       EventMachine.add_timer(0.1) do
         read_and_publish.call
       end
+
     end
   end