ghtorrent 0.8 → 0.8.1

data/lib/ghtorrent/commands/ght_retrieve_dependents.rb

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'amqp'
+require 'json'
+require 'pp'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+
+class GHTRetrieveDependents < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+  include GHTorrent::Persister
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Recursively retrieve all dependent entities for a specific entity
+#{command_name} [options] entity entity-id
+
+#{command_name} entity is one of (in parenthesis the entity-id fields):
+commit        (owner repo sha)
+issue         (owner repo issue_id)
+pull_request  (owner repo pullreq_id)
+#{command_name}
+    BANNER
+
+  end
+
+  REQ_ARGS = {
+      :commit => 3,
+      :issue => 3,
+      :pull_request => 3
+  }
+
+  def logger
+    ghtorrent.logger
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def ghtorrent
+    @gh ||= GHTorrent::Mirror.new(@settings)
+    @gh
+  end
+
+  def db
+    @db ||= ghtorrent.get_db
+    @db
+  end
+
+  def go
+    db
+    type = case ARGV[0]
+             when 'commit'
+               :commit
+             when 'issue'
+               :issue
+             when 'pull_request'
+               :pull_request
+             else
+               Trollop::die("Don't know how to handle #{ARGV[0]}")
+           end
+    unless ARGV.size - 1 == REQ_ARGS[type]
+      Trollop::die("#{ARGV[0]} requires #{REQ_ARGS[type]} arguments")
+    end
+
+    case type
+      when :commit
+        ghtorrent.ensure_commit(ARGV[2], ARGV[3], ARGV[1], true)
+      when :issue
+        ghtorrent.ensure_issue(ARGV[1], ARGV[2], ARGV[3], true, true, true)
+      when :pull_request
+        ghtorrent.ensure_pull_request(ARGV[1], ARGV[2], ARGV[3], true, true, true)
+    end
+
+  end
+end
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -64,12 +64,7 @@ An efficient way to get all data for a single repo
     repo = repo_entry[:name]
 
     def send_message(function, user, repo)
-      begin
-        ght.send(function, user, repo, refresh = true)
-      rescue Exception => e
-        puts STDERR, e.message
-        puts STDERR, e.backtrace
-      end
+      ght.send(function, user, repo, refresh = true)
     end
 
     functions = %w(ensure_commits ensure_forks ensure_pull_requests
@@ -86,67 +81,3 @@ An efficient way to get all data for a single repo
   end
 end
 
-# A version of the GHTorrent class that creates a transaction per processed
-# item
-class TransactedGHTorrent < GHTorrent::Mirror
-
-  def ensure_commit(repo, sha, user, comments = true)
-    check_transaction do
-      super(repo, sha, user, comments)
-    end
-  end
-
-  def ensure_fork(owner, repo, fork_id)
-    check_transaction do
-      super(owner, repo, fork_id)
-    end
-  end
-
-  def ensure_pull_request(owner, repo, pullreq_id,
-      comments = true, commits = true,
-      state = nil, created_at = nil)
-    check_transaction do
-      super(owner, repo, pullreq_id, comments, commits, state, created_at)
-    end
-  end
-
-  def ensure_issue(owner, repo, issue_id, events = true, comments = true, labels = true)
-    check_transaction do
-      super(owner, repo, issue_id, events, comments, labels)
-    end
-  end
-
-  def ensure_project_member(owner, repo, new_member, date_added)
-    check_transaction do
-      super(owner, repo, new_member, date_added)
-    end
-  end
-
-  def ensure_watcher(owner, repo, watcher, date_added = nil)
-    check_transaction do
-      super(owner, repo, watcher, date_added)
-    end
-  end
-
-  def ensure_repo_label(owner, repo, name)
-    check_transaction do
-      super(owner, repo, name)
-    end
-  end
-
-  def check_transaction(&block)
-    begin
-      if @db.in_transaction?
-        yield block
-      else
-        transaction do
-          yield block
-        end
-      end
-    rescue Exception => e
-      puts STDERR, e.message
-      puts STDERR, e.backtrace
-    end
-  end
-
-end

data/lib/ghtorrent/commands/ght_retrieve_repos.rb

@@ -0,0 +1,206 @@
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/retriever'
+
+class GHTRetrieveRepos < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+
+  def logger
+    @logger ||= Logger.new(STDOUT)
+    @logger
+  end
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Retrieve data for multiple repos in parallel. To work, it requires
+a mapping file formatted as follows:
+
+IP UNAME PASSWD NUM_PROCS where
+
+IP = address to use for outgoing requests (use 0.0.0.0 on non-multihomed hosts)
+UNAME = Github user name to use for outgoing requests
+PASSWD = Github password to use for outgoing requests
+NUM_PROCS = Number of processes to spawn for this IP/UNAME combination
+
+Values in the config.yaml file set with the -c command are overriden.
+
+#{command_name} [options] mapping-file
+
+    BANNER
+    options.opt :queue, 'Queue to retrieve project names from',
+                :short => 'q', :default => 'retrieve-repo', :type => :string
+
+  end
+
+  def validate
+    super
+    Trollop::die 'Argument mapping-file is required' unless not args[0].nil?
+  end
+
+  def go
+
+    configs = File.open(ARGV[0]).readlines.map do |line|
+      next if line =~ /^#/
+      ip,name,passwd,instances = line.strip.split(/ /)
+      (1..instances.to_i).map do |i|
+        newcfg = self.settings.clone
+        newcfg = override_config(newcfg, :attach_ip, ip)
+        newcfg = override_config(newcfg, :github_username, name)
+        newcfg = override_config(newcfg, :github_passwd, passwd)
+        newcfg = override_config(newcfg, :mirror_history_pages_back, 1000)
+        newcfg = override_config(newcfg, :mirror_commit_pages_new_repo, 1000)
+        newcfg
+      end
+    end.flatten.select{|x| !x.nil?}
+
+    children = configs.map do |config|
+      pid = Process::fork
+
+      if pid.nil?
+        retriever = GHTRepoRetriever.new(config, options[:queue])
+
+        Signal.trap('TERM') {
+          retriever.stop
+        }
+
+        retriever.run
+        exit
+      else
+        debug "Parent #{Process.pid} forked child #{pid}"
+        pid
+      end
+    end
+
+    debug 'Waiting for children'
+    begin
+      children.each do |pid|
+        debug "Waiting for child #{pid}"
+        Process.waitpid(pid, 0)
+        debug "Child #{pid} exited"
+      end
+    rescue Interrupt
+      debug 'Stopping'
+    end
+  end
+end
+
+class GHTRepoRetriever
+
+  include GHTorrent::Settings
+  include GHTorrent::Retriever
+  include GHTorrent::Persister
+
+  def initialize(config, queue)
+    @config = config
+    @queue = queue
+  end
+
+  def logger
+    ght.logger
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def ext_uniq
+    @ext_uniq ||= config(:uniq_id)
+    @ext_uniq
+  end
+
+  def ght
+    @ght ||= TransactedGhtorrent.new(@config)
+    @ght
+  end
+
+  def settings
+    @config
+  end
+
+  def run
+    AMQP.start(:host => config(:amqp_host),
+               :port => config(:amqp_port),
+               :username => config(:amqp_username),
+               :password => config(:amqp_password)) do |connection|
+
+      connection.on_tcp_connection_loss do |conn, settings|
+        warn 'AMQP: Network failure. Trying to reconnect...'
+        conn.reconnect(false, 2)
+      end
+
+      channel = AMQP::Channel.new(connection)
+      channel.auto_recovery = true
+      channel.prefetch(1)
+
+      channel.on_error do |ch, channel_close|
+        warn 'AMQP: Channel closed. Should reconnect by itself'
+        raise channel_close.reply_text
+      end
+
+      exchange = channel.topic(config(:amqp_exchange), :durable => true,
+                               :auto_delete => false)
+
+      queue = channel.queue(@queue, {:durable => true}).bind(exchange)
+
+      queue.subscribe(:ack => true) do |headers, msg|
+        owner,repo = msg.split(/ /)
+        user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
+
+        if user_entry.nil?
+          warn("Cannot find user #{owner}")
+          headers.ack
+          next
+        end
+
+        repo_entry = ght.transaction { ght.ensure_repo(owner, repo) }
+
+        if repo_entry.nil?
+          warn("Cannot find repository #{owner}/#{repo}")
+          headers.ack
+          next
+        end
+  
+        debug("Retrieving repo #{owner}/#{repo}")
+        def send_message(function, user, repo)
+          ght.send(function, user, repo, refresh = false)
+        end
+
+        functions = %w(ensure_commits ensure_forks ensure_pull_requests
+          ensure_issues ensure_project_members ensure_watchers ensure_labels)
+
+        functions.each do |x|
+
+          begin
+            send_message(x, owner, repo)
+          rescue Interrupt
+            stop
+          rescue Exception
+            warn("Error processing #{x} for #{owner}/#{repo}")
+            next
+          end
+
+          if @stop
+            break
+          end
+        end
+
+        headers.ack
+        debug("Finished processing #{owner}/#{repo}")
+        if @stop
+          connection.disconnect{AMQP.stop { EM.stop }}
+        end
+      end
+    end
+  end
+
+  def stop
+    warn('Stop flag set, waiting for operations to finish')
+    @stop = true
+  end
+end
+

data/lib/ghtorrent/commands/ght_retrieve_user.rb

@@ -42,7 +42,7 @@ An efficient way to get all data for a single user
       end
     end
 
-    functions = %w(ensure_user_followers ensure_orgs)
+    functions = %w(ensure_user_followers ensure_orgs ensure_org)
 
     if ARGV[2].nil?
       functions.each do |x|
@@ -69,4 +69,11 @@ class TransactedGHTorrent < GHTorrent::Mirror
       super(user)
     end
   end
-end
+
+  def ensure_org(user, members = true)
+    check_transaction do
+      super(user, members)
+    end
+  end
+
+end

data/lib/ghtorrent/ghtorrent.rb

@@ -80,9 +80,9 @@ module GHTorrent
     #  [user] The login of the repository owner
     #  [repo] The name of the repository
     #  [comment_id] The login of the member to add
-    def get_commit_comment(user, repo, comment_id)
+    def get_commit_comment(user, repo, sha, comment_id)
       transaction do
-        ensure_commit_comment(user, repo, comment_id)
+        ensure_commit_comment(user, repo, sha, comment_id)
       end
     end
 
@@ -226,8 +226,8 @@ module GHTorrent
                 end
 
       commits.map do |c|
-        ensure_commit(repo, c['sha'], user)
-      end
+        save{ensure_commit(repo, c['sha'], user)}
+      end.select{|x| !x.nil?}
     end
 
     ##
@@ -237,35 +237,37 @@ module GHTorrent
       commits = @db[:commits]
       parents = @db[:commit_parents]
       commit['parents'].map do |p|
-        url = p['url'].split(/\//)
-        this = commits.first(:sha => commit['sha'])
-        parent = commits.first(:sha => url[7])
+        save do
+          url = p['url'].split(/\//)
+          this = commits.first(:sha => commit['sha'])
+          parent = commits.first(:sha => url[7])
+
+          if parent.nil?
+            c = retrieve_commit(url[5], url[7], url[4])
+            if c.nil?
+              warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
+              next
+            end
+            parent = store_commit(c, url[5], url[4])
+          end
 
-        if parent.nil?
-          c = retrieve_commit(url[5], url[7], url[4])
-          if c.nil?
+          if parent.nil?
             warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
             next
           end
-          parent = store_commit(c, url[5], url[4])
-        end
 
-        if parent.nil?
-          warn "GHTorrent: Could not retrieve #{url[4]}/#{url[5]} -> #{url[7]}, parent to commit #{this[:sha]}"
-          next
-        end
+          if parents.first(:commit_id => this[:id],
+                           :parent_id => parent[:id]).nil?
 
-        if parents.first(:commit_id => this[:id],
-                         :parent_id => parent[:id]).nil?
-
-          parents.insert(:commit_id => this[:id],
-                         :parent_id => parent[:id])
-          info "GHTorrent: Added parent #{parent[:sha]} to commit #{this[:sha]}"
-        else
-          debug "GHTorrent: Parent #{parent[:sha]} for commit #{this[:sha]} exists"
+            parents.insert(:commit_id => this[:id],
+                           :parent_id => parent[:id])
+            info "GHTorrent: Added parent #{parent[:sha]} to commit #{this[:sha]}"
+          else
+            debug "GHTorrent: Parent #{parent[:sha]} for commit #{this[:sha]} exists"
+          end
+          parents.first(:commit_id => this[:id], :parent_id => parent[:id])
         end
-        parents.first(:commit_id => this[:id], :parent_id => parent[:id])
-      end
+      end.select{|x| !x.nil?}
     end
 
     ##
@@ -479,7 +481,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_user_follower(followed, x['login']) }
+      end.map { |x| save{ensure_user_follower(followed, x['login']) }}.select{|x| !x.nil?}
     end
 
     ##
@@ -666,7 +668,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_project_member(user, repo, x['login'], time) }
+      end.map { |x| save{ensure_project_member(user, repo, x['login'], time) }}.select{|x| !x.nil?}
     end
 
     ##
@@ -722,7 +724,7 @@ module GHTorrent
     # [user]  The login name of the user to check the organizations for
     #
     def ensure_orgs(user)
-      retrieve_orgs(user).map{|o| ensure_participation(user, o['login'])}
+      retrieve_orgs(user).map{|o| save{ensure_participation(user, o['login'])}}.select{|x| !x.nil?}
     end
 
     ##
@@ -735,6 +737,12 @@ module GHTorrent
     #
     def ensure_participation(user, organization, members = true)
       org = ensure_org(organization, members)
+
+      if org.nil?
+        warn "Organization #{organization} does not exit"
+        return
+      end
+
       usr = ensure_user(user, false, false)
 
       org_members = @db[:organization_members]
@@ -758,22 +766,26 @@ module GHTorrent
     # ==Parameters:
     # [organization]  The login name of the organization
     #
-    def ensure_org(organization, members)
+    def ensure_org(organization, members = true)
       org = @db[:users].first(:login => organization, :type => 'org')
 
       if org.nil?
         org = ensure_user(organization, false, false)
+
+        # Not an organization, don't go ahead
+        if org[:type] != 'ORG'
+          warn "GHTorrent: Account #{organization} is not an organization"
+          return nil
+        end
+
         if members
-        retrieve_org_members(organization).map { |x|
-          ensure_participation(ensure_user(x['login'], false, false)[:login],
-                               organization, false)
-        }
+          retrieve_org_members(organization).map do |x|
+            ensure_participation(ensure_user(x['login'], false, false)[:login],
+                                 organization, false)
+          end
         end
-        org
-      else
-        debug "GHTorrent: Organization #{organization} exists"
-        org
       end
+      org
     end
 
     ##
@@ -796,30 +808,23 @@ module GHTorrent
         end
       end
 
-      not_saved.map{|x| ensure_commit_comment(user, repo, x['id'])}
+      not_saved.map{|x| save{ensure_commit_comment(user, repo, sha, x['id'])}}.select{|x| !x.nil?}
     end
 
-    ##
-    # Get a specific comment
-    #
-    # ==Parameters:
-    # [user]  The login name of the organization
-    # [repo]  The repository containing the commit whose comment will be retrieved
-    # [id]  The comment id to retrieve
-    # [created_at]  The timestamp that the comment was made.
-    def ensure_commit_comment(user, repo, id)
-      stored_comment = @db[:commit_comments].first(:comment_id => id)
+
+    def ensure_commit_comment(owner, repo, sha, comment_id)
+      stored_comment = @db[:commit_comments].first(:comment_id => comment_id)
 
       if stored_comment.nil?
-        retrieved = retrieve_commit_comment(user, repo, id)
+        retrieved = retrieve_commit_comment(owner, repo, sha, comment_id)
 
         if retrieved.nil?
-          warn "GHTorrent: Commit comment #{id} deleted"
+          warn "GHTorrent: Commit comment #{sha}->#{id} deleted"
           return
         end
 
-        commit = ensure_commit(repo, retrieved['commit_id'], user, false)
-        user = ensure_user(user, false, false)
+        commit = ensure_commit(repo, sha, owner, false)
+        user = ensure_user(retrieved['user']['login'], false, false)
         @db[:commit_comments].insert(
             :commit_id => commit[:id],
             :user_id => user[:id],
@@ -830,11 +835,11 @@ module GHTorrent
             :ext_ref_id => retrieved[@ext_uniq],
             :created_at => date(retrieved['created_at'])
         )
-        info "GHTorrent: Added commit comment #{commit[:sha]} -> #{user[:login]}"
+        info "GHTorrent: Added commit comment #{sha} -> #{retrieved['id']} by #{user[:login]}"
       else
-        info "GHTorrent: Commit comment #{id} exists"
+        info "GHTorrent: Commit comment #{sha} -> #{id} exists"
       end
-      @db[:commit_comments].first(:comment_id => id)
+      @db[:commit_comments].first(:comment_id => comment_id)
     end
 
     ##
@@ -859,7 +864,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_watcher(owner, repo, x['login'], nil) }
+      end.map { |x| save{ensure_watcher(owner, repo, x['login'], nil) }}.select{|x| !x.nil?}
     end
 
     ##
@@ -934,7 +939,7 @@ module GHTorrent
                         end
                       end
 
-      raw_pull_reqs.map { |x| ensure_pull_request(owner, repo, x['number']) }
+      raw_pull_reqs.map { |x| save { ensure_pull_request(owner, repo, x['number']) } }.select { |x| !x.nil? }
     end
 
     ##
@@ -966,7 +971,7 @@ module GHTorrent
           info "GHTorrent: New pull request (#{id}) event (#{act}) by (#{actor}) timestamp #{ts}"
         else
           entry.update(:actor_id => user[:id])
-          info "GHTorrent: Pull request (#{id}) history entry (#{act}) timestamp #{ts} exists"
+          info "GHTorrent: Pull request (#{id}) history entry (#{act}) by (#{actor}) timestamp #{ts} exists"
         end
       end
 
@@ -1128,8 +1133,8 @@ module GHTorrent
           acc
         end
       end.map { |x|
-        ensure_pullreq_comment(owner, repo, pullreq_id, x['id'])
-      }
+        save{ensure_pullreq_comment(owner, repo, pullreq_id, x['id'])}
+      }.select{|x| !x.nil?}
     end
 
     def ensure_pullreq_comment(owner, repo, pullreq_id, comment_id)
@@ -1191,26 +1196,28 @@ module GHTorrent
         return
       end
 
-      retrieve_pull_req_commits(owner, repo, pullreq_id).reduce([]){|acc, c|
+      retrieve_pull_req_commits(owner, repo, pullreq_id).reduce([]) { |acc, c|
         next if c.nil?
         head_repo_owner = c['url'].split(/\//)[4]
         head_repo_name = c['url'].split(/\//)[5]
         x = ensure_commit(head_repo_name, c['sha'], head_repo_owner, true)
         acc << x if not x.nil?
         acc
-      }.map { |c|
-        exists = @db[:pull_request_commits].first(:pull_request_id => pullreq[:id],
-                                                  :commit_id => c[:id])
-        if exists.nil?
-          @db[:pull_request_commits].insert(:pull_request_id => pullreq[:id],
-                                            :commit_id => c[:id])
-
-          info "GHTorrent: Added commit #{c[:sha]} to pullreq #{owner}/#{repo} -> #{pullreq_id}"
-        else
-          debug "GHTorrent: Commit #{c[:sha]} exists in pullreq #{owner}/#{repo} -> #{pullreq_id}"
-          exists
+      }.map do |c|
+        save do
+          exists = @db[:pull_request_commits].first(:pull_request_id => pullreq[:id],
+                                                    :commit_id => c[:id])
+          if exists.nil?
+            @db[:pull_request_commits].insert(:pull_request_id => pullreq[:id],
+                                              :commit_id => c[:id])
+
+            info "GHTorrent: Added commit #{c[:sha]} to pullreq #{owner}/#{repo} -> #{pullreq_id}"
+          else
+            debug "GHTorrent: Commit #{c[:sha]} exists in pullreq #{owner}/#{repo} -> #{pullreq_id}"
+            exists
+          end
         end
-      }
+      end.select{|x| !x.nil?}
     end
 
     ##
@@ -1241,7 +1248,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_fork(owner, repo, x['id']) }
+      end.map { |x| save{ensure_fork(owner, repo, x['id']) }}.select{|x| !x.nil?}
     end
 
     ##
@@ -1288,7 +1295,7 @@ module GHTorrent
                      end
                    end
 
-      raw_issues.map { |x| ensure_issue(owner, repo, x['number']) }
+      raw_issues.map { |x| save { ensure_issue(owner, repo, x['number']) } }.select { |x| !x.nil? }
     end
 
     ##
@@ -1379,8 +1386,8 @@ module GHTorrent
           acc
         end
       end.map { |x|
-        ensure_issue_event(owner, repo, issue_id, x['id'])
-      }
+        save{ensure_issue_event(owner, repo, issue_id, x['id'])}
+      }.select{|x| !x.nil?}
     end
 
     ##
@@ -1491,8 +1498,8 @@ module GHTorrent
           acc
         end
       end.map { |x|
-        ensure_issue_comment(owner, repo, issue_id, x['id'], pull_req_id)
-      }
+        save{ensure_issue_comment(owner, repo, issue_id, x['id'], pull_req_id)}
+      }.select{|x| !x.nil?}
     end
 
     ##
@@ -1555,7 +1562,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_repo_label(owner, repo, x['name']) }
+      end.map { |x| save { ensure_repo_label(owner, repo, x['name']) } }.select { |x| !x.nil? }
     end
 
     ##
@@ -1613,7 +1620,7 @@ module GHTorrent
         else
           acc
         end
-      end.map { |x| ensure_issue_label(owner, repo, issue[:issue_id], x['name']) }
+      end.map { |x| save{ensure_issue_label(owner, repo, issue[:issue_id], x['name']) }}.select{|x| !x.nil?}
 
     end
 
@@ -1677,6 +1684,20 @@ module GHTorrent
       end
     end
 
+    def save(&block)
+      if config(:rescue_loops) == 'true'
+        begin
+          yield block
+        rescue Exception => e
+          @logger.error e.message
+          @logger.error e.backtrace.join("\n")
+          nil
+        end
+      else
+        yield block
+      end
+    end
+
     private
 
     # Store a commit contained in a hash. First check whether the commit exists.