ghtorrent 0.10 → 0.11

data/lib/ghtorrent/logging.rb

@@ -1,9 +1,15 @@
 require 'logger'
 
+require 'ghtorrent/settings'
+
 module GHTorrent
   module Logging
 
-    DEBUG_LEVEL = defined?(Logger) ? Logger::DEBUG : 0
+    include GHTorrent::Settings
+
+    def error(msg)
+      log(:warn, msg)
+    end
 
     def warn(msg)
       log(:warn, msg)
@@ -17,30 +23,77 @@ module GHTorrent
       log(:debug, msg)
     end
 
+    # Default logger
+    def loggerr
+      @logger ||= proc do
+        @logger_uniq ||= config(:logging_uniq)
+
+        logger = if config(:logging_file).casecmp('stdout')
+                   Logger.new(STDOUT)
+                 elsif config(:logging_file).casecmp('stderr')
+                   Logger.new(STDERR)
+                 else
+                   Logger.new(config(:logging_file))
+                 end
+
+        logger.level =
+            case config(:logging_level).downcase
+              when 'debug' then
+                Logger::DEBUG
+              when 'info' then
+                Logger::INFO
+              when 'warn' then
+                Logger::WARN
+              when 'error' then
+                Logger::ERROR
+              else
+                Logger::INFO
+            end
+
+        logger.formatter = proc do |severity, time, progname, msg|
+          if progname.nil? or progname.empty?
+            progname = @logger_uniq
+          end
+          "#{severity}, #{time.iso8601}, #{progname} -- #{msg}\n"
+        end
+        logger
+      end.call
+
+      @logger
+    end
+
     private
 
+    def retrieve_caller
+      @logprefixes ||= Hash.new
+
+      c = caller[2]
+      unless @logprefixes.key? c
+        file_path = c.split(/:/)[0]
+        @logprefixes[c] = File.basename(file_path) + ': '
+      end
+
+      @logprefixes[c]
+
+    end
+
     # Log a message at the given level.
     def log(level, msg)
 
       case level
         when :fatal then
-          logger.fatal msg
+          loggerr.fatal (retrieve_caller + msg)
         when :error then
-          logger.error msg
+          loggerr.error (retrieve_caller + msg)
         when :warn then
-          logger.warn msg
+          loggerr.warn  (retrieve_caller + msg)
         when :info then
-          logger.info msg
+          loggerr.info  (retrieve_caller + msg)
         when :debug then
-          logger.debug msg
+          loggerr.debug (retrieve_caller + msg)
         else
-          logger.debug msg
+          loggerr.debug (retrieve_caller + msg)
       end
     end
-
-    # Default logger
-    def logger
-      @logger ||= Logger.new(STDOUT)
-    end
   end
 end

data/lib/ghtorrent/migrations/014_add_deleted_to_projects.rb

@@ -10,7 +10,7 @@ Sequel.migration do
                :default => false
 
     puts 'Field deleted added'
-    puts 'Remember to run the fixes/update_deleted.rb script to mark deleted projects'
+    puts 'Remember to run the fixes/ght_update_project_info.rb script to mark deleted projects'
   end
 
   down do

data/lib/ghtorrent/migrations/019_add_fake_to_users.rb

@@ -19,7 +19,7 @@ Sequel.migration do
                   and not exists (select * from project_members where users.id = user_id)
                   and not exists (select * from issues where reporter_id=users.id )
                   and not exists (select * from issues where assignee_id=users.id )
-                a nd not exists (select * from organization_members where user_id = users.id);"
+                  and not exists (select * from organization_members where user_id = users.id);"
       end
     end
   end

data/lib/ghtorrent/migrations/020_add_deleted_to_users.rb

@@ -0,0 +1,19 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+
+Sequel.migration do
+
+  up do
+    puts 'Add column deleted to users'
+      add_column :users, :deleted, TrueClass, :null => false, :default => false
+  end
+
+  down do
+    puts 'Drop column deleted from users'
+    alter_table :users do
+      drop_column :deleted
+    end
+  end
+end

data/lib/ghtorrent/migrations/021_remove_ext_ref_id.rb

@@ -0,0 +1,42 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+
+Sequel.migration do
+
+  up do
+    puts 'Dropping column ext_ref_id from all tables'
+
+    drop_column :users, :ext_ref_id
+    drop_column :projects, :ext_ref_id
+    drop_column :commits, :ext_ref_id
+    drop_column :followers, :ext_ref_id
+    drop_column :commit_comments, :ext_ref_id
+    drop_column :pull_request_history, :ext_ref_id
+    drop_column :pull_request_comments, :ext_ref_id
+    drop_column :issues, :ext_ref_id
+    drop_column :issue_events, :ext_ref_id
+    drop_column :issue_comments, :ext_ref_id
+    drop_column :repo_labels, :ext_ref_id
+    drop_column :repo_milestones, :ext_ref_id
+    drop_column :watchers, :ext_ref_id
+  end
+
+  down do
+    puts 'Add column ext_ref_id to all tables'
+    add_column :users, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :projects, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :commits, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :followers, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :commit_comments, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :pull_request_history, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :pull_request_comments, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :issues, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :issue_events, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :issue_comments, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :repo_labels, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :repo_milestones, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    add_column :watchers, :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+  end
+end

data/lib/ghtorrent/migrations/022_add_project_languages.rb

@@ -0,0 +1,24 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+
+Sequel.migration do
+
+  up do
+    puts 'Adding table project_languages'
+
+    create_table :project_languages do
+      foreign_key :project_id, :projects, :null => false
+      String :language, :null => false
+      Integer :lines, :null => false, :default => 0
+      DateTime :created_at, :null => false,
+               :default => Sequel::CURRENT_TIMESTAMP
+    end
+  end
+
+  down do
+    puts 'Dropping table project_languages'
+    drop_table :project_languages
+  end
+end

data/lib/ghtorrent/multiprocess_queue_client.rb

@@ -1,3 +1,28 @@
+# A generic class to implement a multiprocess client for clients that listen
+# to the same queue for incoming messages.
+#
+# Client creation is driven by an input file whose format is described
+# in #options.banner.
+#
+# MultiprocessQueueClient expects overriding classes to overload the #clazz method
+# with the name of a class that respects the following protocol:
+#
+# @example
+#   class ExampleQueueClient
+#     # Constructor accepting a GHTorrent::Settings and a String object
+#     # representing the client configuration and the queue to connect to
+#     def initialize(config, queue); end
+#
+#     # Start processing using the specified GHTorrent::Command as input
+#     def run(command); end
+#
+#     # Stop processing of queue messages
+#     def stop; end
+#   end
+#
+# MultiprocessQueueClient does not do any error handling/respawning of
+# dead children. Client/Sub classes are expected to handle errors by
+# themselves.
 class MultiprocessQueueClient < GHTorrent::Command
 
   include GHTorrent::Settings
@@ -33,11 +58,6 @@ Values in the config.yaml file set with the -c command are overridden.
                 :type => :string
   end
 
-  def logger
-    @logger ||= Logger.new(STDOUT)
-    @logger
-  end
-
   def validate
     super
     Trollop::die 'Argument mapping-file is required' unless not args[0].nil?

data/lib/ghtorrent/retriever.rb

@@ -4,7 +4,7 @@ require 'cgi'
 require 'ghtorrent/api_client'
 require 'ghtorrent/settings'
 require 'ghtorrent/utils'
-require 'ghtorrent/gh_torrent_exception'
+require 'ghtorrent/logging'
 
 module GHTorrent
   module Retriever
@@ -12,10 +12,7 @@ module GHTorrent
     include GHTorrent::Settings
     include GHTorrent::Utils
     include GHTorrent::APIClient
-
-    def ext_uniq
-      raise Exception.new("Unimplemented")
-    end
+    include GHTorrent::Logging
 
     def persister
       raise Exception.new("Unimplemented")
@@ -32,13 +29,12 @@ module GHTorrent
         end
 
         unq = persister.store(:users, u)
-        u[ext_uniq] = unq
         what = user_type(u['type'])
-        info "Retriever: New #{what} #{user}"
+        info "Added user #{what} #{user}"
         u
       else
         what = user_type(stored_user.first['type'])
-        debug "Retriever: Already got #{what} #{user}"
+        debug "#{what} #{user} exists"
         stored_user.first
       end
     end
@@ -72,22 +68,23 @@ module GHTorrent
             if not email.nil? and user['email'] == email
               user
             else
+              warn "Could not find user #{email}"
               nil
             end
           else
+            warn "Could not find user #{email}"
             nil
           end
         end
       else
         unless byemail['user']['login'].nil?
-          info "Retriever: User #{byemail['user']['login']} retrieved by email #{email}"
+          info "Added user #{byemail['user']['login']} retrieved by email #{email}"
           retrieve_user_byusername(byemail['user']['login'])
         else
           u = byemail['user']
           unq = persister.store(:users, u)
-          u[ext_uniq] = unq
           what = user_type(u['type'])
-          info "Retriever: New #{what} #{user}"
+          info "Added user #{what} #{user}"
           u
         end
       end
@@ -110,19 +107,52 @@ module GHTorrent
         x['follows'] = user
 
         exists = !persister.find(:followers, {'follows' => user,
-                                     'login' => x['login']}).empty?
+                                              'login' => x['login']}).empty?
 
         if not exists
           persister.store(:followers, x)
-          info "Retriever: Added follower #{user} -> #{x['login']}"
+          info "Added follower #{user} -> #{x['login']}"
         else
-          debug "Retriever: Follower #{user} -> #{x['login']} exists"
+          debug "Follower #{user} -> #{x['login']} exists"
         end
       end
 
       persister.find(:followers, {'follows' => user})
     end
 
+    def retrieve_user_following(user)
+      following = paged_api_request(ghurl "users/#{user}/following")
+      user_followers_entry = nil
+
+      following.each do |x|
+
+        if user_followers_entry.nil?
+          reverse_lookup = persister.find(:followers, {'follows' => x['login'],
+                                                       'login' => user})
+          if reverse_lookup.empty?
+            user_followers_entry = retrieve_user_followers(x['login']).\
+                                     find{|y| y['login'] == user}
+          else
+            user_followers_entry = reverse_lookup[0]
+          end
+        end
+
+        exists = !persister.find(:followers, {'follows' => x['login'],
+                                              'login' => user}).empty?
+        if not exists
+          user_followers_entry['follows'] = x['login']
+          user_followers_entry.delete(:_id)
+          user_followers_entry.delete('_id')
+          a = persister.store(:followers, user_followers_entry)
+          info "Added following #{user} -> #{x['login']}"
+        else
+          debug "Following #{user} -> #{x['login']} exists"
+        end
+      end
+
+      persister.find(:followers, {'login' => user})
+    end
+
     # Retrieve a single commit from a repo
     def retrieve_commit(repo, sha, user)
       commit = persister.find(:commits, {'sha' => "#{sha}"})
@@ -136,11 +166,10 @@ module GHTorrent
         end
 
         unq = persister.store(:commits, c)
-        info "Retriever: New commit #{user}/#{repo} -> #{sha}"
-        c[ext_uniq] = unq
+        info "Added commit #{user}/#{repo} -> #{sha}"
         c
       else
-        debug "Retriever: Already got commit #{user}/#{repo} -> #{sha}"
+        debug "Commit #{user}/#{repo} -> #{sha} exists"
         commit.first
       end
     end
@@ -173,15 +202,18 @@ module GHTorrent
         end
 
         unq = persister.store(:repos, r)
-        info "Retriever: New repo #{user} -> #{repo}"
-        r[ext_uniq] = unq
+        info "Added repo #{user} -> #{repo}"
         r
       else
-        debug "Retriever: Already got repo #{user} -> #{repo}"
+        debug "Repo #{user} -> #{repo} exists"
         stored_repo.first
       end
     end
 
+    def retrieve_languages(owner, repo)
+      paged_api_request ghurl "repos/#{owner}/#{repo}/languages"
+    end
+
     # Retrieve organizations the provided user participates into
     def retrieve_orgs(user)
       url = ghurl "users/#{user}/orgs"
@@ -208,9 +240,9 @@ module GHTorrent
 
         if not exists
           persister.store(:org_members, x)
-          info "Retriever: Added org member #{org} -> #{x['login']}"
+          info "Added org_member #{org} -> #{x['login']}"
         else
-          debug "Retriever: Org Member #{org} -> #{x['login']} exists"
+          debug "Org Member #{org} -> #{x['login']} exists"
         end
       end
 
@@ -239,15 +271,15 @@ module GHTorrent
         r = api_request(ghurl "repos/#{owner}/#{repo}/comments/#{id}")
 
         if r.empty?
-          debug "Retriever: Commit comment #{id} deleted"
+          warn "Could not find commit_comment #{id}. Deleted?"
           return
         end
 
         persister.store(:commit_comments, r)
-        info "Retriever: Added commit comment #{r['commit_id']} -> #{r['id']}"
+        info "Added commit_comment #{r['commit_id']} -> #{r['id']}"
         persister.find(:commit_comments, {'commit_id' => sha, 'id' => id}).first
       else
-        debug "Retriever: Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
+        debug "Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
         comment
       end
     end
@@ -334,7 +366,7 @@ module GHTorrent
       url = review_comments_url
       retrieved_comments = paged_api_request url
 
-      retrieved_comments.each { |x|
+      retrieved_comments.each do |x|
         x['owner'] = owner
         x['repo'] = repo
         x['pullreq_id'] = pullreq_id.to_i
@@ -345,7 +377,7 @@ module GHTorrent
                                                    'id' => x['id']}).empty?
           persister.store(:pull_request_comments, x)
         end
-      }
+      end
 
       persister.find(:pull_request_comments, {'owner' => owner, 'repo' => repo,
                                               'pullreq_id' => pullreq_id})
@@ -354,13 +386,13 @@ module GHTorrent
     def retrieve_pull_req_comment(owner, repo, pullreq_id, comment_id)
       comment = persister.find(:pull_request_comments, {'repo' => repo,
                                                  'owner' => owner,
-                                                 'pullreq_id' => pullreq_id,
+                                                 'pullreq_id' => pullreq_id.to_i,
                                                  'id' => comment_id}).first
       if comment.nil?
         r = api_request(ghurl "repos/#{owner}/#{repo}/pulls/comments/#{comment_id}")
 
         if r.empty?
-          debug "Retriever: Pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id} deleted"
+          warn "Could not find pullreq_comment #{owner}/#{repo} #{pullreq_id}->#{comment_id}. Deleted?"
           return
         end
 
@@ -368,12 +400,12 @@ module GHTorrent
         r['owner'] = owner
         r['pullreq_id'] = pullreq_id.to_i
         persister.store(:pull_request_comments, r)
-        info "Retriever: Added pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id}"
+        info "Added pullreq_comment #{owner}/#{repo} #{pullreq_id}->#{comment_id}"
         persister.find(:pull_request_comments, {'repo' => repo, 'owner' => owner,
-                                         'pullreq_id' => pullreq_id,
+                                         'pullreq_id' => pullreq_id.to_i,
                                          'id' => comment_id}).first
       else
-        debug "Retriever: Pullreq comment #{owner}/#{repo} #{pullreq_id}->#{comment_id} exists"
+        debug "Pull request comment #{owner}/#{repo} #{pullreq_id}->#{comment_id} exists"
         comment
       end
     end
@@ -409,11 +441,11 @@ module GHTorrent
                                           'repo' => repo,
                                           'issue_id' => issue_id,
                                           'id' => x['id']}).empty?
-          info "Retriever: Added issue event #{owner}/#{repo} #{issue_id}->#{x['id']}"
+          info "Added issue_event #{owner}/#{repo} #{issue_id}->#{x['id']}"
           persister.store(:issue_events, x)
         end
         x
-      }.map {|y| y[ext_uniq] = '0'; y}
+      }
       a = persister.find(:issue_events, {'owner' => owner, 'repo' => repo,
                                          'issue_id' => issue_id})
       if a.empty? then issue_events else a end
@@ -428,7 +460,7 @@ module GHTorrent
         r = api_request(ghurl "repos/#{owner}/#{repo}/issues/events/#{event_id}")
 
         if r.empty?
-          warn "Retriever: Issue event #{owner}/#{repo} #{issue_id}->#{event_id} deleted"
+          warn "Could not find issue_event #{owner}/#{repo} #{issue_id}->#{event_id}. Deleted?"
           return
         end
 
@@ -436,13 +468,13 @@ module GHTorrent
         r['owner'] = owner
         r['issue_id'] = issue_id
         persister.store(:issue_events, r)
-        info "Retriever: Added issue event #{owner}/#{repo} #{issue_id}->#{event_id}"
+        info "Added issue_event #{owner}/#{repo} #{issue_id}->#{event_id}"
         a = persister.find(:issue_events, {'repo' => repo, 'owner' => owner,
                                        'issue_id' => issue_id,
                                        'id' => event_id}).first
-        if a.nil? then r[ext_uniq] = '0'; r else a end
+        if a.nil? then r else a end
       else
-        debug "Retriever: Issue event #{owner}/#{repo} #{issue_id}->#{event_id} exists"
+        debug "Issue event #{owner}/#{repo} #{issue_id}->#{event_id} exists"
         event
       end
     end
@@ -463,7 +495,7 @@ module GHTorrent
           persister.store(:issue_comments, x)
         end
         x
-      }.map {|y| y[ext_uniq] = '0'; y}
+      }
       a = persister.find(:issue_comments, {'owner' => owner, 'repo' => repo,
                                            'issue_id' => issue_id})
       if a.empty? then comments else a end
@@ -478,7 +510,7 @@ module GHTorrent
         r = api_request(ghurl "repos/#{owner}/#{repo}/issues/comments/#{comment_id}")
 
         if r.empty?
-          warn "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} deleted"
+          warn "Could not find issue_comment #{owner}/#{repo} #{issue_id}->#{comment_id}. Deleted?"
           return
         end
 
@@ -486,13 +518,13 @@ module GHTorrent
         r['owner'] = owner
         r['issue_id'] = issue_id
         persister.store(:issue_comments, r)
-        info "Retriever: Added issue comment #{owner}/#{repo} #{issue_id}->#{comment_id}"
+        info "Added issue_comment #{owner}/#{repo} #{issue_id}->#{comment_id}"
         a = persister.find(:issue_comments, {'repo' => repo, 'owner' => owner,
                                          'issue_id' => issue_id,
                                          'id' => comment_id}).first
-        if a.nil? then r[ext_uniq] = '0'; r else a end
+        if a.nil? then r else a end
       else
-        debug "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} exists"
+        debug "Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} exists"
         comment
       end
     end
@@ -521,19 +553,23 @@ module GHTorrent
       api_request "https://api.github.com/events"
     end
 
-    # Get all events for the specified repo
+    # Get all events for the specified repo.
+    # GitHub will only return 90 days of events
     def get_repo_events(owner, repo)
       url = ghurl("repos/#{owner}/#{repo}/events")
       r = paged_api_request(url)
 
       r.each do |e|
-        if get_event(e['id']).empty?
-          info "Retriever: Already got event #{owner}/#{repo} -> #{e['id']}"
+        unless get_event(e['id']).empty?
+          debug "Repository event #{owner}/#{repo} -> #{e['type']}-#{e['id']} already exists"
         else
-          @persister.store(:events, e)
-          info "Retriever: Added event #{owner}/#{repo} -> #{e['id']}"
+          persister.store(:events, e)
+          info "Added event for repository #{owner}/#{repo} -> #{e['type']}-#{e['id']}"
         end
       end
+
+      persister.find(:events, {'repo.name' => "#{owner}/#{repo}"})
+
     end
 
     # Get a specific event by +id+.
@@ -575,8 +611,11 @@ module GHTorrent
             exists = !instances.empty?
 
             unless exists
+              x = api_request(x['url'])
+              x['repo'] = repo
+              x['owner'] = user
               persister.store(entity, x)
-              info "Retriever: Added #{entity} #{user}/#{repo} -> #{x[discriminator]}"
+              info "Added #{entity} #{user}/#{repo} -> #{x[discriminator]}"
             else
               if refresh
                 instances.each do |i|
@@ -590,10 +629,10 @@ module GHTorrent
                   instance_selector = selector.merge({discriminator => id})
                   persister.del(entity, instance_selector)
                   persister.store(entity, x)
-                  debug "Retriever: Refreshing #{entity} #{user}/#{repo} -> #{x[discriminator]}"
+                  debug "Refreshing #{entity} #{user}/#{repo} -> #{x[discriminator]}"
                 end
               else
-                debug "Retriever: #{entity} #{user}/#{repo} -> #{x[discriminator]} exists"
+                debug "#{entity} #{user}/#{repo} -> #{x[discriminator]} exists"
               end
             end
 
@@ -621,15 +660,19 @@ module GHTorrent
     end
 
     def repo_bound_item(user, repo, item_id, entity, url, selector,
-        discriminator, order = :asc)
+                        discriminator, order = :asc)
       stored_item = repo_bound_instance(entity, selector, discriminator, item_id)
 
-      if stored_item.empty?
-        repo_bound_items(user, repo, entity, url, selector, discriminator,
-                         item_id, false, order).first
-      else
-        stored_item.first
+      r = if stored_item.empty?
+            repo_bound_items(user, repo, entity, url, selector, discriminator,
+                             item_id, false, order).first
+          else
+            stored_item.first
+          end
+      if r.nil?
+        warn "Could not find #{entity} #{user}/#{repo} -> #{item_id}. Deleted?"
       end
+      r
     end
 
     def repo_bound_instance(entity, selector, discriminator, item_id)