ghtorrent 0.10 → 0.11

data/lib/ghtorrent/commands/ght_retrieve_dependents.rb

@@ -4,7 +4,6 @@ require 'pp'
 
 require 'ghtorrent/ghtorrent'
 require 'ghtorrent/settings'
-require 'ghtorrent/logging'
 require 'ghtorrent/command'
 
 class GHTRetrieveDependents < GHTorrent::Command
@@ -33,10 +32,6 @@ pull_request  (owner repo pullreq_id)
       :pull_request => 3
   }
 
-  def logger
-    ghtorrent.logger
-  end
-
   def persister
     @persister ||= connect(:mongo, settings)
     @persister

data/lib/ghtorrent/commands/ght_retrieve_one.rb

@@ -38,13 +38,8 @@ Retrieve just one item
     @persister
   end
 
-  def ext_uniq
-    @ext_uniq ||= config(:uniq_id)
-    @ext_uniq
-  end
-
   def ght
-    @ght ||= TransactedGhtorrent.new(settings)
+    @ght ||= TransactedGHTorrent.new(settings)
     @ght
   end
 

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -2,7 +2,6 @@ require 'rubygems'
 
 require 'ghtorrent/ghtorrent'
 require 'ghtorrent/settings'
-require 'ghtorrent/logging'
 require 'ghtorrent/command'
 require 'ghtorrent/retriever'
 
@@ -11,6 +10,7 @@ class GHTRetrieveRepo < GHTorrent::Command
   include GHTorrent::Settings
   include GHTorrent::Retriever
   include GHTorrent::Persister
+  include GHTorrent::EventProcessing
 
   def prepare_options(options)
     options.banner <<-BANNER
@@ -19,35 +19,54 @@ An efficient way to get all data for a single repo
 #{command_name} [options] owner repo
 
     BANNER
+    options.opt :no_events, 'Skip retrieving events', :default => false
+    options.opt :no_entities, 'Skip retrieving entities', :default => false
+
+    options.opt :only_stage, "Only do the provided stage of entity retrieval (one of: #{stages.join(',')})",
+                :type => String
+    options.opt :exclude_events, 'Comma separated list of event types to exclude from processing',
+                :type => String
+    options.opt :events_after, 'Process all events later than the provided event id',
+                :type => Integer
+    options.opt :events_before, 'Process all events earlier than the provided event id',
+                :type => Integer
   end
 
   def validate
     super
-    Trollop::die "Two arguments are required" unless args[0] && !args[0].empty?
+    Trollop::die 'Two arguments are required' unless args[0] && !args[0].empty?
+
+    unless options[:exclude_events].nil?
+      @exclude_event_types = options[:exclude_events].split(/,/)
+    else
+      @exclude_event_types = []
+    end
+
+    unless options[:only_stage].nil?
+      Trollop::die("Not a valid function: #{options[:only_stage]}") unless stages.include? options[:only_stage]
+    end
+
   end
 
-  def logger
-    ght.logger
+  def stages
+    %w(ensure_commits ensure_forks ensure_pull_requests
+       ensure_issues ensure_watchers ensure_labels ensure_languages) #ensure_project_members
   end
 
+
   def persister
     @persister ||= connect(:mongo, settings)
     @persister
   end
 
-  def ext_uniq
-    @ext_uniq ||= config(:uniq_id)
-    @ext_uniq
-  end
-
-  def ght
-    @ght ||= TransactedGhtorrent.new(settings)
-    @ght
+  def ghtorrent
+    @ghtorrent ||= TransactedGHTorrent.new(settings)
+    @ghtorrent
   end
 
   def go
     self.settings = override_config(settings, :mirror_history_pages_back, 1000)
-    user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
+    user_entry = ghtorrent.transaction{ghtorrent.ensure_user(ARGV[0], false, false)}
 
     if user_entry.nil?
       Trollop::die "Cannot find user #{ARGV[0]}"
@@ -55,7 +74,7 @@ An efficient way to get all data for a single repo
 
     user = user_entry[:login]
 
-    repo_entry = ght.transaction{ght.ensure_repo(ARGV[0], ARGV[1])}
+    repo_entry = ghtorrent.transaction{ghtorrent.ensure_repo(ARGV[0], ARGV[1])}
 
     if repo_entry.nil?
       Trollop::die "Cannot find repository #{ARGV[0]}/#{ARGV[1]}"
@@ -63,21 +82,32 @@ An efficient way to get all data for a single repo
 
     repo = repo_entry[:name]
 
-    def send_message(function, user, repo)
-      ght.send(function, user, repo)
+    unless options[:no_entities_given]
+      if options[:only_stage].nil?
+        stages.each do |x|
+          ghtorrent.send(x, user, repo)
+        end
+      else
+        ghtorrent.send(options[:only_stage], user, repo)
+      end
     end
 
-    functions = %w(ensure_commits ensure_forks ensure_pull_requests
-       ensure_issues ensure_project_members ensure_watchers ensure_labels)
-
-    if ARGV[2].nil?
-      functions.each do |x|
-        send_message(x, user, repo)
+    # Process repo events
+    unless options[:no_events_given]
+      events = get_repo_events(ARGV[0], ARGV[1]).sort{|e| e['id'].to_i}
+      events.each do |event|
+        begin
+          next if @exclude_event_types.include? event['type']
+          next if options[:events_after_given] and event['id'].to_i <= options[:events_after]
+          next if options[:events_before_given] and event['id'].to_i >= options[:events_before]
+
+          send(event['type'], event)
+          puts "Processed event #{event['type']}-#{event['id']}"
+        rescue StandardError => e
+           puts "Could not process event #{event['type']}-#{event['id']}: #{e.message}"
+        end
       end
-    else
-      Trollop::die("Not a valid function: #{ARGV[2]}") unless functions.include? ARGV[2]
-      send_message(ARGV[2], user, repo)
     end
+
   end
 end
-

data/lib/ghtorrent/commands/ght_retrieve_repos.rb

@@ -28,13 +28,8 @@ class GHTRepoRetriever
     ght.logger
   end
 
-  def ext_uniq
-    @ext_uniq ||= config(:uniq_id)
-    @ext_uniq
-  end
-
   def ght
-    @ght ||= TransactedGhtorrent.new(@config)
+    @ght ||= TransactedGHTorrent.new(@config)
     @ght
   end
 
@@ -58,7 +53,7 @@ class GHTRepoRetriever
         warn("Trying to get user #{owner}, attempt #{i}")
         begin
           user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
-        rescue Exception => e
+        rescue StandardError => e
           warn e.message
         end
       end
@@ -69,11 +64,7 @@ class GHTRepoRetriever
       end
 
       repo_entry = ght.transaction { ght.ensure_repo(owner, repo,
-                                                     commits = false,
-                                                     project_members = false,
-                                                     watchers = false,
-                                                     forks = false,
-                                                     labels = false) }
+                                                     recursive = false) }
 
       if repo_entry.nil?
         warn("Cannot find repository #{owner}/#{repo}")
@@ -83,8 +74,7 @@ class GHTRepoRetriever
       debug("Retrieving repo #{owner}/#{repo}")
 
       retrieval_stages = %w(ensure_commits ensure_forks ensure_pull_requests
-                            ensure_issues ensure_project_members
-                            ensure_watchers ensure_labels)
+                            ensure_issues ensure_watchers ensure_labels) # ensure_project_members
 
       retrieval_stages.each do |x|
         run_retrieval_stage(ght, owner, repo, x)
@@ -114,7 +104,7 @@ class GHTRepoRetriever
       else
         ght.send(function, owner, repo)
       end
-    rescue Exception => e
+    rescue StandardError => e
       warn("Error processing #{function} for #{owner}/#{repo}")
       warn("Exception message #{$!}")
       warn("Exception trace #{e.backtrace.join("\n")}")

data/lib/ghtorrent/commands/ght_retrieve_user.rb

@@ -2,19 +2,21 @@ require 'rubygems'
 
 require 'ghtorrent/ghtorrent'
 require 'ghtorrent/settings'
-require 'ghtorrent/logging'
 require 'ghtorrent/command'
 require 'ghtorrent/retriever'
-require 'ghtorrent/transacted_ghtorrent'
-require 'ghtorrent/commands/ght_retrieve_repo'
+require 'ghtorrent/transacted_gh_torrent'
+require 'ghtorrent/commands/full_user_retriever'
 
-class GHTRetrieveUser < GHTRetrieveRepo
+class GHTRetrieveUser < GHTorrent::Command
+
+  include GHTorrent::Retriever
+  include GHTorrent::Commands::FullUserRetriever
 
   def prepare_options(options)
     options.banner <<-BANNER
 An efficient way to get all data for a single user
 
-#{command_name} [options] user
+#{command_name} [options] login
 
     BANNER
   end
@@ -24,57 +26,14 @@ An efficient way to get all data for a single user
     Trollop::die "One argument is required" unless args[0] && !args[0].empty?
   end
 
-  def go
-    self.settings = override_config(settings, :mirror_history_pages_back, -1)
-    user_entry = ght.transaction{ght.ensure_user(ARGV[0], false, false)}
-
-    if user_entry.nil?
-      Trollop::die "Cannot find user #{ARGV[0]}"
-    end
-
-    user = user_entry[:login]
-
-    def send_message(function, user)
-      begin
-        ght.send(function, user)
-      rescue Exception => e
-        puts STDERR, e.message
-        puts STDERR, e.backtrace
-      end
-    end
-
-    functions = %w(ensure_user_followers ensure_orgs ensure_org)
-
-    if ARGV[1].nil?
-      functions.each do |x|
-        send_message(x, user)
-      end
-    else
-      Trollop::die("Not a valid function: #{ARGV[1]}") unless functions.include? ARGV[1]
-      send_message(ARGV[1], user)
-    end
-
-  end
-end
-
-class TransactedGhtorrent 
-
-  def ensure_user_followers(user)
-    check_transaction do
-      super(user)
-    end
+  def ght
+    @ght ||= TransactedGHTorrent.new(settings)
+    @ght
   end
 
-  def ensure_orgs(user)
-    check_transaction do
-      super(user)
-    end
-  end
-
-  def ensure_org(user, members = true)
-    check_transaction do
-      super(user, members)
-    end
+  def go
+    login = ARGV[0]
+    retrieve_user(login)
   end
 
 end

data/lib/ghtorrent/commands/ght_retrieve_users.rb

@@ -0,0 +1,49 @@
+require 'ghtorrent/retriever'
+require 'ghtorrent/transacted_gh_torrent'
+require 'ghtorrent/commands/full_user_retriever'
+
+# Retrieve user information en masse
+class GHTRetrieveUsers < MultiprocessQueueClient
+
+  def clazz
+    GHTUserRetriever
+  end
+
+end
+
+# Initialize a user retrieval process
+class GHTUserRetriever
+
+  include GHTorrent::Retriever
+  include GHTorrent::Commands::FullUserRetriever
+
+  attr_accessor :ght
+
+  def initialize(config, queue)
+    @config = config
+    @queue = queue
+  end
+
+  def settings
+    @config
+  end
+
+  def run(command)
+
+    processor = Proc.new do |user|
+      @ght ||= TransactedGHTorrent.new(@config)
+
+      retrieve_user(user)
+    end
+
+    command.queue_client(@queue, :after, processor)
+
+  end
+
+  def stop
+    warn('Stop flag set, waiting for operations to finish')
+    @stop = true
+  end
+
+end
+

data/lib/ghtorrent/commands/ght_update_repo.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+require 'ghtorrent'
+
+class GHTUpdateRepo < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Retriever
+  include GHTorrent::Persister
+  include GHTorrent::Logging
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Updates the deleted field in the project table with current data
+
+#{command_name} owner repo
+
+    BANNER
+  end
+
+  def validate
+    super
+    Trollop::die "Takes two arguments" if ARGV.size == 1
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def db
+    @db ||= @ght.get_db
+  end
+
+  def date(arg)
+    if arg.class != Time
+      Time.parse(arg)#.to_i
+    else
+      arg
+    end
+  end
+
+  def set_deleted(owner, repo)
+    db.from(:projects, :users).\
+       where(:projects__owner_id => :users__id).\
+       where(:users__login => owner).\
+       where(:projects__name => repo).\
+       update(:projects__deleted => true)
+    info("Project #{owner}/#{repo} marked as deleted")
+  end
+
+  def update_mysql(owner, repo, retrieved)
+
+    parent = unless retrieved['parent'].nil?
+               @ght.ensure_repo(retrieved['parent']['owner']['login'],
+                                retrieved['parent']['name'])
+             end
+
+    db.from(:projects, :users).\
+       where(:projects__owner_id => :users__id).\
+       where(:users__login => owner).\
+       where(:projects__name => repo).\
+       update(
+         :projects__url => retrieved['url'],
+         :projects__description => retrieved['description'],
+         :projects__language => retrieved['language'],
+         :projects__created_at => date(retrieved['created_at']),
+         :projects__forked_from => unless parent.nil? then parent[:id] end)
+    debug("Repo #{owner}/#{repo} updated")
+
+    @ght.ensure_languages(owner, repo)
+  end
+
+  def process_project(owner, name)
+    @ght.transaction do
+
+      in_mongo = persister.find(:repos, {'owner.login' => owner, 'name' => name })
+      on_github = api_request(ghurl ("repos/#{owner}/#{name}"))
+
+      unless in_mongo.empty? and on_github.empty?
+        in_mysql = retrieve_repo(owner, name)
+      end
+
+      if in_mongo.empty?
+        if on_github.empty?
+          if in_mysql.nil?
+            # Project does not exist anywhere
+            warn "Repo #{owner}/#{name} does not exist in MySQL"
+          else
+            # Project exists in MySQL but not on Github or Mongo
+            # Mark it as deleted
+            set_deleted(owner, name)
+          end
+        else
+          # Project does not exist in Mongo, but exists in Github
+          if in_mysql.nil?
+            warn "Repo #{owner}/#{name} does not exist in MySQL"
+          else
+            # The retrieval process already added it to Mongo, so update MySQL
+            update_mysql(owner, name, in_mysql)
+          end
+        end
+      else
+        if on_github.empty?
+          # Project was deleted on Github. Mark it as deleted.
+          set_deleted(owner, name)
+        else
+          update_mysql(owner, name, in_mysql)
+        end
+      end
+    end
+  end
+
+  def go
+
+    @ght ||= GHTorrent::Mirror.new(settings)
+
+    unless ARGV[1].nil?
+      process_project(ARGV[0], ARGV[1])
+      exit(0)
+    end
+
+  end
+end
+
+GHTUpdateRepo.run