ghtorrent 0.5 → 0.6

data/bin/ght-process-event

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ghtorrent'
+
+
+class GHTProcessEvent < GHTDataRetrieval
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Process one or more event ids
+#{command_name} [options] eventid [...]
+    BANNER
+
+  end
+
+  def go
+    ARGV.each do |a|
+      data = ghtorrent.get_event(a)
+      unless data.empty?
+        event = data[0]
+        begin
+          send(event['type'], event)
+        rescue Exception => e
+          puts e
+          puts e.backtrace
+        end
+      else
+        warn "GHTProcessEvent: No event with id #{a}"
+      end
+    end
+  end
+end
+
+GHTProcessEvent.run

data/bin/ght-retrieve-repo

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ghtorrent'
+
+GHTRetrieveRepo.run(ARGV)

data/bin/ght-rm-dupl

@@ -1,134 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require 'mongo'
+require 'ghtorrent'
 
-require 'ghtorrent/settings'
-require 'ghtorrent/logging'
-require 'ghtorrent/command'
-require 'ghtorrent/persister'
-
-class GHRMDupl < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Persister
-
-  def col_info()
-    {
-        :commits => {
-            :unq => "sha",
-            :col => persister.get_underlying_connection.collection(:commits.to_s),
-        },
-        :events => {
-            :unq => "id",
-            :col => persister.get_underlying_connection.collection(:events.to_s),
-        }
-    }
-  end
-
-  def persister
-    @persister ||= connect(:mongo, @settings)
-    @persister
-  end
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Removes duplicate entries from collections
-
-#{command_name} [options] collection
-
-#{command_name} options:
-    BANNER
-
-    options.opt :earliest, 'Seconds since epoch of earliest item to load',
-                :short => 'e', :default => 0, :type => :int
-    options.opt :snapshot, 'Perform clean up every x records',
-                :short => 's', :default => -1, :type => :int
-  end
-
-  def validate
-    super
-    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
-  end
-
-  # Print MongoDB remove statements that
-  # remove all but one entries for each commit.
-  def remove_duplicates(data, col)
-    removed = 0
-    data.select { |k, v| v.size > 1 }.each do |k, v|
-      v.slice(0..(v.size - 2)).map do |x|
-        removed += 1 if delete_by_id col, x
-      end
-    end
-    removed
-  end
-
-  def delete_by_id(col, id)
-    begin
-      col.remove({'_id' => id})
-      true
-    rescue Mongo::OperationFailure
-      puts "Cannot remove record with id #{id} from #{col.name}"
-      false
-    end
-  end
-
-  def go
-    collection = case ARGV[0]
-              when "commits" then
-                :commits
-              when "events" then
-                :events
-              else
-                puts "Not a known collection name: #{ARGV[0]}\n"
-            end
-
-    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
-
-    snapshot = options[:snapshot]
-
-    puts "Deleting duplicates from collection #{collection}"
-    puts "Deleting duplicates after #{Time.at(options[:earliest])}"
-    puts "Perform clean up every #{snapshot} records"
-
-    # Various counters to report stats
-    processed = total_processed = removed = 0
-
-    data = Hash.new
-
-    # The following code needs to save intermediate results to cope
-    # with large datasets
-    col_info[collection][:col].find(from, :fields => col_info[collection][:unq]).each do |r|
-      _id = r["_id"]
-      commit = read_value(r, col_info[collection][:unq])
-
-      # If entries cannot be parsed, remove them
-      if commit.empty?
-        puts "Deleting unknown entry #{_id}"
-        removed += 1 if delete_by_id col_info[collection][:col], _id
-      else
-        data[commit] = [] if data[commit].nil?
-        data[commit] << _id
-      end
-
-      processed += 1
-      total_processed += 1
-
-      print "\rProcessed #{processed} records"
-
-      # Calculate duplicates, save intermediate result
-      if snapshot > 0 and processed > snapshot
-        puts "\nLoaded #{data.size} values, cleaning"
-        removed += remove_duplicates data, col_info[collection][:col]
-        data = Hash.new
-        processed = 0
-      end
-    end
-
-    removed += remove_duplicates data, col_info[collection][:col]
-
-    puts "\nProcessed #{total_processed}, deleted #{removed} duplicates"
-  end
-end
-
-GHRMDupl.run
+GHRMDupl.run

data/lib/ghtorrent.rb

@@ -48,3 +48,13 @@ require 'ghtorrent/retriever'
 
 # SQL database fillup methods
 require 'ghtorrent/ghtorrent'
+
+# Commands
+require 'ghtorrent/commands/ght_data_retrieval'
+require 'ghtorrent/commands/ght_mirror_events'
+require 'ghtorrent/commands/ght_get_more_commits'
+require 'ghtorrent/commands/ght_rm_dupl'
+require 'ghtorrent/commands/ght_load'
+require 'ghtorrent/commands/ght_retrieve_repo'
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -4,7 +4,7 @@ module GHTorrent
 
     ENTITIES = [:users, :commits, :followers, :repos, :events, :org_members,
         :commit_comments, :repo_collaborators, :watchers, :pull_requests,
-        :forks, :pull_request_comments, :issue_comments, :issues
+        :forks, :pull_request_comments, :issue_comments, :issues, :issue_events
     ]
 
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -110,8 +110,12 @@ module GHTorrent
           get_collection("forks")
         when :pull_request_comments
           get_collection("pull_request_comments")
+        when :issues
+          get_collection("issues")
         when :issue_comments
           get_collection("issue_comments")
+        when :issue_events
+          get_collection("issue_events")
       end
     end
 
@@ -130,7 +134,7 @@ module GHTorrent
                    Mongo::ReplSetConnection.new(repl_arr, :read => :secondary)\
                                            .db(config(:mongo_db))
                  end
-        init_db(@mongo) if @mongo.collections.size <= 0
+        init_db(@mongo) if @mongo.collections.size < ENTITIES.size
         @mongo
       else
         @mongo
@@ -183,6 +187,13 @@ module GHTorrent
       ensure_index(:pull_request_comments, "owner")
       ensure_index(:pull_request_comments, "pullreq_id")
       ensure_index(:pull_request_comments, "id")
+      ensure_index(:issues, "repo")
+      ensure_index(:issues, "owner")
+      ensure_index(:issues, "issue_id")
+      ensure_index(:issue_events, "repo")
+      ensure_index(:issue_events, "owner")
+      ensure_index(:issue_events, "issue_id")
+      ensure_index(:issue_events, "id")
     end
 
     def rescue_connection_failure(max_retries=60)

data/lib/ghtorrent/api_client.rb

@@ -8,12 +8,14 @@ require 'ghtorrent/logging'
 require 'ghtorrent/settings'
 require 'ghtorrent/time'
 require 'ghtorrent/cache'
+require 'version'
 
 module GHTorrent
   module APIClient
     include GHTorrent::Logging
     include GHTorrent::Settings
     include GHTorrent::Cache
+    include GHTorrent::Logging
 
     # This is to fix an annoying bug in JRuby's SSL not being able to
     # verify a valid certificate.
@@ -25,10 +27,20 @@ module GHTorrent
     # result pages.
     def paged_api_request(url, pages = -1, cache = true, last = nil)
 
-      data = if URI.parse(url).query.nil? # Top level request, no params
-               api_request_raw(url, false)
-             else
+      url = if not url.include?("per_page")
+              if url.include?("?")
+                url + "&per_page=100"
+              else
+                url + "?per_page=100"
+              end
+            else
+              url
+            end
+
+      data = if CGI::parse(URI::parse(url).query).has_key?("page")
                api_request_raw(url, use_cache?(cache, method = :paged))
+             else
+               api_request_raw(url, false)
              end
 
       return [] if data.nil?
@@ -79,7 +91,7 @@ module GHTorrent
                         when "prod"
                           :prod
                         else
-                          raise GHTorrentException("")
+                          raise GHTorrentException.new("Don't know cache configuration #{@cache_mode}")
                       end
       case @cache_mode
         when :dev
@@ -128,21 +140,22 @@ module GHTorrent
     # Do the actual request and return the result object
     def api_request_raw(url, use_cache = false)
       @num_api_calls ||= 0
-      @ts ||= Time.now().tv_sec()
+      @ts ||= Time.now.to_i
+      @started_min ||= Time.now.min
 
       #Rate limiting to avoid error requests
       if Time.now().tv_sec() - @ts < 60 then
         if @num_api_calls >= @settings['mirror']['reqrate'].to_i
-          sleep = 60 - (Time.now().tv_sec() - @ts)
+          sleep = 60 - (Time.now.to_i - @ts)
           debug "APIClient: Sleeping for #{sleep}"
           sleep (sleep)
           @num_api_calls = 0
-          @ts = Time.now().tv_sec()
+          @ts = Time.now.to_i
         end
       else
         debug "APIClient: Tick, num_calls = #{@num_api_calls}, zeroing"
         @num_api_calls = 0
-        @ts = Time.now().tv_sec()
+        @ts = Time.now.to_i
       end
 
       begin
@@ -166,7 +179,17 @@ module GHTorrent
             end
 
         total = Time.now.to_ms - start_time.to_ms
-        debug "APIClient: Request: #{url} (#{@num_api_calls} calls,#{if from_cache then " from cache," end} Total: #{total} ms)"
+        debug "APIClient: Request: #{url} (#{@num_api_calls} calls #{if from_cache then " from cache," else "(#{contents.meta['x-ratelimit-remaining']} remaining)," end} Total: #{total} ms)"
+
+        if not from_cache and config(:respect_api_ratelimit) and
+            contents.meta['x-ratelimit-remaining'].to_i < 400
+          sleep = 60 - @started_min
+          debug "APIClient: Request limit reached, sleeping for #{sleep} min"
+          sleep(sleep * 60)
+          @started_min = Time.now.min
+          @num_api_calls = 0
+        end
+
         contents
       rescue OpenURI::HTTPError => e
         case e.io.status[0].to_i
@@ -176,10 +199,10 @@ module GHTorrent
               403, # Forbidden
               404, # Not found
               422 then # Unprocessable entity
-            STDERR.puts "#{url}: #{e.io.status[1]}"
+            warn "#{url}: #{e.io.status[1]}"
             return nil
           else # Server error or HTTP conditions that Github does not report
-            STDERR.puts "#{url}"
+            warn "#{url}"
             raise e
         end
       end
@@ -187,12 +210,23 @@ module GHTorrent
 
     def do_request(url)
       @attach_ip ||= config(:attach_ip)
+      @username ||= config(:github_username)
+      @passwd ||= config(:github_passwd)
+      @user_agent ||= "ghtorrent-v#{GHTorrent::VERSION}"
+
+      @open_func ||= if @username.nil?
+        lambda {|url| open(url, 'User-Agent' => @user_agent)}
+      else
+        lambda {|url| open(url,
+                           'User-Agent' => @user_agent,
+                           :http_basic_authentication => [@username, @passwd])}
+      end
 
       if @attach_ip.nil? or @attach_ip.eql? "0.0.0.0"
-        open(url)
+        @open_func.call(url)
       else
         attach_to(@attach_ip) do
-          open(url)
+          @open_func.call(url)
         end
       end
     end

data/lib/ghtorrent/bson_orderedhash.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'bson'
 
 class BSON::OrderedHash
 
@@ -20,4 +21,4 @@ class BSON::OrderedHash
   def to_json
     to_h.to_json
   end
-end
+end

data/lib/ghtorrent/command.rb

@@ -4,6 +4,7 @@ require 'daemons'
 require 'etc'
 
 require 'ghtorrent/settings'
+require 'version'
 
 module GHTorrent
 
@@ -16,6 +17,7 @@ module GHTorrent
   class Command
 
     include GHTorrent::Settings
+    include GHTorrent::Settings
 
     # Specify the run method for subclasses.
     class << self
@@ -33,6 +35,8 @@ module GHTorrent
         command.process_options
         command.validate
 
+        puts "GHTorrent version: #{GHTorrent::VERSION}"
+
         command.settings = YAML::load_file command.options[:config]
 
         unless command.options[:addr].nil?
@@ -41,6 +45,18 @@ module GHTorrent
                                                      command.options[:addr])
         end
 
+        unless command.options[:username].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :github_username,
+                                                     command.options[:username])
+        end
+
+        unless command.options[:password].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :github_passwd,
+                                                     command.options[:password])
+        end
+
         if command.options[:daemon]
           if Process.uid == 0
             # Daemonize as a proper system daemon
@@ -97,6 +113,8 @@ Standard options:
         opt :daemon, 'run as daemon', :short => 'd'
         opt :user, 'run as the specified user (only when started as root)',
             :short => 'u', :type => String
+        opt :username, 'Username at Github', :type => String
+        opt :password, 'Password at Github', :type => String
       end
     end
 

data/lib/ghtorrent/commands/ght_data_retrieval.rb

@@ -0,0 +1,218 @@
+require 'rubygems'
+require 'amqp'
+require 'json'
+require 'pp'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+
+class GHTDataRetrieval < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+
+  def parse(msg)
+    JSON.parse(msg)
+  end
+
+  def PushEvent(data)
+    data['payload']['commits'].each do |c|
+      url = c['url'].split(/\//)
+
+      ghtorrent.get_commit url[4], url[5], url[7]
+    end
+  end
+
+  def WatchEvent(data)
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    watcher = data['actor']['login']
+    created_at = data['created_at']
+
+    ghtorrent.get_watcher owner, repo, watcher, created_at
+  end
+
+  def FollowEvent(data)
+    follower = data['actor']['login']
+    followed = data['payload']['target']['login']
+    created_at = data['created_at']
+
+    ghtorrent.get_follower(follower, followed, created_at)
+  end
+
+  def MemberEvent(data)
+    owner = data['actor']['login']
+    repo = data['repo']['name'].split(/\//)[1]
+    new_member = data['payload']['member']['login']
+    created_at = data['created_at']
+
+    ghtorrent.get_project_member(owner, repo, new_member, created_at)
+  end
+
+  def CommitCommentEvent(data)
+    user = data['actor']['login']
+    repo = data['repo']['name'].split(/\//)[1]
+    id = data['payload']['comment']['id']
+    created_at = data['created_at']
+
+    ghtorrent.get_commit_comment(user, repo, id, created_at)
+  end
+
+  def PullRequestEvent(data)
+    owner = data['payload']['pull_request']['base']['repo']['owner']['login']
+    repo = data['payload']['pull_request']['base']['repo']['name']
+    pullreq_id = data['payload']['number']
+    action = data['payload']['action']
+    created_at = data['created_at']
+
+    ghtorrent.get_pull_request(owner, repo, pullreq_id, action, created_at)
+  end
+
+  def ForkEvent(data)
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    fork_id = data['payload']['forkee']['id']
+    created_at = data['created_at']
+
+    ghtorrent.get_fork(owner, repo, fork_id, created_at)
+  end
+
+  def PullRequestReviewCommentEvent(data)
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    comment_id = data['payload']['comment']['id']
+    pullreq_id = data['payload']['comment']['_links']['pull_request']['href'].split(/\//)[-1]
+    created_at = data['created_at']
+
+    ghtorrent.get_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
+  end
+
+  def IssuesEvent(data)
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    issue_id = data['payload']['issue']['number']
+    created_at = data['created_at']
+
+    ghtorrent.get_issue(owner, repo, issue_id, created_at)
+  end
+
+  def IssueCommentEvent(data)
+    owner = data['repo']['name'].split(/\//)[0]
+    repo = data['repo']['name'].split(/\//)[1]
+    issue_id = data['payload']['issue']['number']
+    comment_id = data['payload']['comment']['id']
+    created_at = data['created_at']
+
+    ghtorrent.get_issue_comment(owner, repo, issue_id, comment_id)
+  end
+
+  def handlers
+    %w(PushEvent WatchEvent FollowEvent MemberEvent
+        CommitCommentEvent PullRequestEvent ForkEvent
+        PullRequestReviewCommentEvent IssuesEvent IssueCommentEvent)
+    #%w(IssuesEvent IssueCommentEvent)
+  end
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Retrieves events from queues and processes them through GHTorrent
+#{command_name} [options]
+
+#{command_name} options:
+    BANNER
+
+    options.opt :filter,
+                'Only process messages for repos in the provided file',
+                :short => 'f', :type => String
+  end
+
+  def validate
+    super
+    Trollop::die "Filter file does not exist" if options[:filter] and not File.exist?(options[:filter])
+  end
+
+  def logger
+    ghtorrent.logger
+  end
+
+  def ghtorrent
+    @gh ||= GHTorrent::Mirror.new(@settings)
+    @gh
+  end
+
+  def go
+    filter = Array.new
+
+    if options[:filter]
+      File.open(options[:filter]).each { |l|
+        next if l.match(/^ *#/)
+        parts = l.split(/ /)
+        next if parts.size < 2
+        debug "GHTDataRetrieval: Filtering events by #{parts[0] + "/" + parts[1]}"
+        filter << parts[0] + "/" + parts[1]
+      }
+    end
+
+    # Graceful exit
+    Signal.trap('INT') {
+      info "GHTDataRetrieval: Received SIGINT, exiting"
+      AMQP.stop { EM.stop }
+    }
+    Signal.trap('TERM') {
+      info "GHTDataRetrieval: Received SIGTERM, exiting"
+      AMQP.stop { EM.stop }
+    }
+
+    AMQP.start(:host => config(:amqp_host),
+               :port => config(:amqp_port),
+               :username => config(:amqp_username),
+               :password => config(:amqp_password)) do |connection|
+
+      channel = AMQP::Channel.new(connection, :prefetch => config(:amqp_prefetch))
+      exchange = channel.topic(config(:amqp_exchange), :durable => true,
+                               :auto_delete => false)
+
+      handlers.each { |h|
+        queue = channel.queue("#{h}s", {:durable => true})\
+                       .bind(exchange, :routing_key => "evt.#{h}")
+
+        info "GHTDataRetrieval: Binding handler #{h} to routing key evt.#{h}"
+
+        queue.subscribe(:ack => true) do |headers, msg|
+          begin
+            data = parse(msg)
+            info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
+
+            unless options[:filter].nil?
+              if filter.include?(data['repo']['name'])
+                send(h, data)
+              else
+                info "GHTDataRetrieval: Repo #{data['repo']['name']} not in process list. Ignoring event #{data['type']}-#{data['id']}"
+              end
+            else
+              send(h, data)
+            end
+            headers.ack
+            info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
+          rescue Exception => e
+            # Give a message a chance to be reprocessed
+            if headers.redelivered?
+              data = parse(msg)
+              warn "GHTDataRetrieval: Could not process event: #{data['type']}-#{data['id']}"
+              headers.reject(:requeue => false)
+            else
+              headers.reject(:requeue => true)
+            end
+
+            STDERR.puts e
+            STDERR.puts e.backtrace.join("\n")
+          end
+        end
+      }
+    end
+  end
+end
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :