ghtorrent 0.4 → 0.5

data/bin/ght-mirror-events

@@ -1,11 +1,18 @@
+#!/usr/bin/env ruby
+
 require 'rubygems'
 require 'yaml'
 require 'amqp'
 require 'eventmachine'
-require 'ghtorrent'
 require 'json'
 require 'logger'
 
+require 'ghtorrent/api_client'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/persister'
+require 'ghtorrent/command'
+
 class GHTMirrorEvents < GHTorrent::Command
 
   include GHTorrent::Settings
@@ -13,12 +20,8 @@ class GHTMirrorEvents < GHTorrent::Command
   include GHTorrent::Persister
   include GHTorrent::APIClient
 
-  attr_reader :settings
-
-  def initialize(args)
-    super(args)
-    @args = args
-    @name = self.class.name
+  def logger
+    @logger
   end
 
   def store_count(events)
@@ -42,7 +45,7 @@ class GHTMirrorEvents < GHTorrent::Command
   def retrieve(exchange)
     begin
       new = dupl = 0
-      events = api_request "https://api.github.com/events"
+      events = api_request "https://api.github.com/events", false
       (new, dupl, stored) = store_count events
 
       # This means that first page cannot contain all new events. Go
@@ -67,23 +70,17 @@ class GHTMirrorEvents < GHTorrent::Command
     end
   end
 
-  def prepare_options(options)
-    @name = "ght-mirror-events"
-  end
-
   def go
-    @gh = GHTorrent::Mirror.new(options[:config])
-    @settings = @gh.settings
     @persister = connect(:mongo, @settings)
     @logger = Logger.new(STDOUT)
 
     # Graceful exit
     Signal.trap('INT') {
-      info ("Received SIGINT, exiting")
+      info "Received SIGINT, exiting"
       AMQP.stop { EM.stop }
     }
     Signal.trap('TERM') {
-      info ("Received SIGTERM, exiting")
+      info "Received SIGTERM, exiting"
       AMQP.stop { EM.stop }
     }
 

data/bin/ght-rm-dupl

@@ -1,92 +1,134 @@
+#!/usr/bin/env ruby
+
 require 'rubygems'
 require 'mongo'
-require 'ghtorrent-old'
-
-GH = Mirror.new
-GH.init("config.yaml")
-
-# Unique keys per known collection
-per_col = {
-    :commits => {
-        :payload => "commit.id",
-        :col => GH.commits_col,
-    },
-    :events => {
-        :payload => "id",
-        :col => GH.events_col,
+
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/persister'
+
+class GHRMDupl < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Persister
+
+  def col_info()
+    {
+        :commits => {
+            :unq => "sha",
+            :col => persister.get_underlying_connection.collection(:commits.to_s),
+        },
+        :events => {
+            :unq => "id",
+            :col => persister.get_underlying_connection.collection(:events.to_s),
+        }
     }
-}
-
-# Print MongoDB remove statements that
-# remove all but one entries for each commit.
-def remove_duplicates(data, col)
-  removed = 0
-  data.select { |k, v| v.size > 1 }.each do |k, v|
-    v.slice(0..(v.size - 2)).map do |x|
-      removed += 1 if delete_by_id col, x
-    end
   end
-  removed
-end
 
-def delete_by_id(col, id)
-  begin
-    col.remove({'_id' => id})
-    true
-  rescue Mongo::OperationFailure
-    puts "Cannot remove record with id #{id} from #{col.name}"
-    false
+  def persister
+    @persister ||= connect(:mongo, @settings)
+    @persister
   end
-end
 
-which = case ARGV[0]
-          when "commits" then :commits
-          when "events" then :events
-          else puts "Not a known collection name: #{ARGV[0]}\n"
-        end
-
-from = case ARGV[1]
-         when nil then {}
-         else
-           t = Time.at(ARGV[1].to_i)
-           STDERR.puts "Searching for duplicates after #{t}"
-           {'_id' => {'$gte' => BSON::ObjectId.from_time(t)}}
-       end
-
-# Various counters to report stats
-processed = total_processed = removed = 0
-
-data = Hash.new
-
-# The following code needs to save intermediate results to cope
-# with large datasets
-per_col[which][:col].find(from, :fields => per_col[which][:payload]).each do |r|
-  _id = r["_id"]
-  commit = GH.read_value(r, per_col[which][:payload])
-
-  # If entries cannot be parsed, remove them
-  if commit.empty?
-    puts "Deleting unknown entry #{_id}"
-    removed += 1 if delete_by_id per_col[which][:col], _id
-  else
-    data[commit] = [] if data[commit].nil?
-    data[commit] << _id
+  def prepare_options(options)
+    options.banner <<-BANNER
+Removes duplicate entries from collections
+
+#{command_name} [options] collection
+
+#{command_name} options:
+    BANNER
+
+    options.opt :earliest, 'Seconds since epoch of earliest item to load',
+                :short => 'e', :default => 0, :type => :int
+    options.opt :snapshot, 'Perform clean up every x records',
+                :short => 's', :default => -1, :type => :int
   end
 
-  processed += 1
-  total_processed += 1
+  def validate
+    super
+    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
+  end
 
-  print "\rProcessed #{processed} records"
+  # Print MongoDB remove statements that
+  # remove all but one entries for each commit.
+  def remove_duplicates(data, col)
+    removed = 0
+    data.select { |k, v| v.size > 1 }.each do |k, v|
+      v.slice(0..(v.size - 2)).map do |x|
+        removed += 1 if delete_by_id col, x
+      end
+    end
+    removed
+  end
+
+  def delete_by_id(col, id)
+    begin
+      col.remove({'_id' => id})
+      true
+    rescue Mongo::OperationFailure
+      puts "Cannot remove record with id #{id} from #{col.name}"
+      false
+    end
+  end
+
+  def go
+    collection = case ARGV[0]
+              when "commits" then
+                :commits
+              when "events" then
+                :events
+              else
+                puts "Not a known collection name: #{ARGV[0]}\n"
+            end
+
+    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
+
+    snapshot = options[:snapshot]
+
+    puts "Deleting duplicates from collection #{collection}"
+    puts "Deleting duplicates after #{Time.at(options[:earliest])}"
+    puts "Perform clean up every #{snapshot} records"
+
+    # Various counters to report stats
+    processed = total_processed = removed = 0
 
-  # Calculate duplicates, save intermediate result
-  if processed > 500000
-    puts "\nLoaded #{data.size} values, cleaning"
-    removed += remove_duplicates data, per_col[which][:col]
     data = Hash.new
-    processed = 0
+
+    # The following code needs to save intermediate results to cope
+    # with large datasets
+    col_info[collection][:col].find(from, :fields => col_info[collection][:unq]).each do |r|
+      _id = r["_id"]
+      commit = read_value(r, col_info[collection][:unq])
+
+      # If entries cannot be parsed, remove them
+      if commit.empty?
+        puts "Deleting unknown entry #{_id}"
+        removed += 1 if delete_by_id col_info[collection][:col], _id
+      else
+        data[commit] = [] if data[commit].nil?
+        data[commit] << _id
+      end
+
+      processed += 1
+      total_processed += 1
+
+      print "\rProcessed #{processed} records"
+
+      # Calculate duplicates, save intermediate result
+      if snapshot > 0 and processed > snapshot
+        puts "\nLoaded #{data.size} values, cleaning"
+        removed += remove_duplicates data, col_info[collection][:col]
+        data = Hash.new
+        processed = 0
+      end
+    end
+
+    removed += remove_duplicates data, col_info[collection][:col]
+
+    puts "\nProcessed #{total_processed}, deleted #{removed} duplicates"
   end
 end
 
-removed += remove_duplicates data, per_col[which][:col]
-
-puts "Processed #{total_processed}, deleted #{removed} duplicates"
+GHRMDupl.run

data/lib/ghtorrent.rb

@@ -1,6 +1,4 @@
 module GHTorrent
-  VERSION = '0.4'
-
   # Route keys used for setting up queues for events, using GHTorrent
   ROUTEKEY_CREATE = "evt.CreateEvent"
   ROUTEKEY_DELETE = "evt.DeleteEvent"
@@ -22,19 +20,31 @@ module GHTorrent
 
 end
 
-require 'ghtorrent/command'
+# Shared extensions to library methods
+require 'ghtorrent/hash'
+require 'ghtorrent/time'
+require 'ghtorrent/bson_orderedhash'
 
+# Basic utility modules
+require 'version'
+require 'ghtorrent/gh_torrent_exception'
 require 'ghtorrent/utils'
 require 'ghtorrent/logging'
 require 'ghtorrent/settings'
+require 'ghtorrent/cache'
 require 'ghtorrent/api_client'
-require 'ghtorrent/call_stack'
 
+# Support for command line utilities offered by this gem
+require 'ghtorrent/command'
+
+# Configuration and drivers for caching retrieved data
 require 'ghtorrent/adapters/base_adapter'
 require 'ghtorrent/adapters/mongo_persister'
 require 'ghtorrent/adapters/noop_persister'
 
+# Support for retrieving and saving intermediate results
 require 'ghtorrent/persister'
 require 'ghtorrent/retriever'
 
+# SQL database fillup methods
 require 'ghtorrent/ghtorrent'

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -3,13 +3,14 @@ module GHTorrent
   class BaseAdapter
 
     ENTITIES = [:users, :commits, :followers, :repos, :events, :org_members,
-        :commit_comments, :repo_collaborators, :watchers
+        :commit_comments, :repo_collaborators, :watchers, :pull_requests,
+        :forks, :pull_request_comments, :issue_comments, :issues
     ]
 
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.
     def store(entity, data = {})
       unless ENTITIES.include?(entity)
-        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise GHTorrentException.new("Perister: Entity #{entity} not known")
       end
     end
 
@@ -50,14 +51,14 @@ module GHTorrent
     # matching JSON object.
     def find(entity, query = {})
       unless ENTITIES.include?(entity)
-        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise GHTorrentException.new("Perister: Entity #{entity} not known")
       end
     end
 
     # Find the record identified by +id+ in +entity+
     def find_by_ext_ref_id(entity, id)
       unless ENTITIES.include?(entity)
-        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise GHTorrentException.new("Perister: Entity #{entity} not known")
       end
     end
 
@@ -65,8 +66,19 @@ module GHTorrent
     # The +query+ can be any query supported by +find+.
     def count(entity, query = {})
       unless ENTITIES.include?(entity)
-        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise GHTorrentException.new("Perister: Entity #{entity} not known")
       end
     end
+
+    # Get a raw connection to the underlying data store. The connection is
+    # implementaiton dependent.
+    def get_underlying_connection
+      raise "Unimplemented"
+    end
+
+    # Close the current connection and release any held resources
+    def close
+      raise "Unimplemented"
+    end
   end
 end

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -1,4 +1,6 @@
 require 'mongo'
+require 'ghtorrent/adapters/base_adapter'
+require 'ghtorrent/bson_orderedhash'
 
 module GHTorrent
 
@@ -14,7 +16,8 @@ module GHTorrent
         :mongo_port => "mongo.port",
         :mongo_db => "mongo.db",
         :mongo_username => "mongo.username",
-        :mongo_passwd => "mongo.password"
+        :mongo_passwd => "mongo.password",
+        :mongo_replicas => "mongo.replicas"
     }
 
     attr_reader :settings
@@ -27,47 +30,21 @@ module GHTorrent
 
       @settings = set
       @uniq = config(:uniq_id)
-      @mongo = Mongo::Connection.new(config(:mongo_host),
-                                     config(:mongo_port))\
-                                .db(config(:mongo_db))
-      @enttodb = {
-          :users => get_collection("users"),
-          :commits => get_collection("commits"),
-          :repos => get_collection("repos"),
-          :followers => get_collection("followers"),
-          :events => get_collection("events"),
-          :org_members => get_collection("org_members"),
-          :commit_comments => get_collection("commit_comments"),
-          :repo_collaborators => get_collection("repo_collaborators"),
-          :watchers => get_collection("watchers")
-      }
-
-      # Ensure that the necessary indexes exist
-      ensure_index(:events, "id")
-      ensure_index(:users, "login")
-      ensure_index(:commits, "sha")
-      ensure_index(:repos, "name")
-      ensure_index(:followers, "follows")
-      ensure_index(:org_members, "org")
-      ensure_index(:commit_comments, "repo")
-      ensure_index(:commit_comments, "user")
-      ensure_index(:commit_comments, "commit_id")
-      ensure_index(:repo_collaborators, "repo")
-      ensure_index(:repo_collaborators, "owner")
-      ensure_index(:repo_collaborators, "login")
-      ensure_index(:watchers, "repo")
-      ensure_index(:watchers, "owner")
-      ensure_index(:watchers, "login")
     end
 
     def store(entity, data = {})
       super
-      get_entity(entity).insert(data).to_s
+      rescue_connection_failure do
+        get_entity(entity).insert(data).to_s
+      end
     end
 
     def find(entity, query = {})
       super
-      result = get_entity(entity).find(query)
+      result = rescue_connection_failure do
+        get_entity(entity).find(query)
+      end
+
       result.to_a.map { |r|
         r[@uniq] = r['_id'].to_s;
         r.to_h
@@ -83,27 +60,86 @@ module GHTorrent
     # Count the number of items returned by +query+
     def count(entity, query)
       super
-      get_entity(entity).count(:query => query)
+      rescue_connection_failure do
+        get_entity(entity).count(:query => query)
+      end
+    end
+
+    def get_underlying_connection
+      mongo
+    end
+
+    def close
+      unless @mongo.nil?
+        @mongo.close if @mongo.class == Mongo::ReplSetConnection
+        @mongo.connection.close if @mongo.class == Mongo::Connection
+
+        @mongo = nil
+      end
     end
 
     private
 
     def get_collection(col)
-      @mongo.collection(col.to_s)
+      mongo.collection(col.to_s)
     end
 
     def get_entity(entity)
-      col = @enttodb[entity]
+      case entity
+        when :users
+          get_collection("users")
+        when :commits
+          get_collection("commits")
+        when :repos
+          get_collection("repos")
+        when :followers
+          get_collection("followers")
+        when :org_members
+          get_collection("org_members")
+        when :events
+          get_collection("events")
+        when :commit_comments
+          get_collection("commit_comments")
+        when :repo_collaborators
+          get_collection("repo_collaborators")
+        when :watchers
+          get_collection("watchers")
+        when :pull_requests
+          get_collection("pull_requests")
+        when :forks
+          get_collection("forks")
+        when :pull_request_comments
+          get_collection("pull_request_comments")
+        when :issue_comments
+          get_collection("issue_comments")
+      end
+    end
 
-      if col.nil?
-        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
+    def mongo
+      if @mongo.nil?
+
+        replicas = config(:mongo_replicas)
+
+        @mongo = if replicas.nil?
+                   Mongo::Connection.new(config(:mongo_host),
+                                         config(:mongo_port))\
+                                    .db(config(:mongo_db))
+                 else
+                   repl_arr = replicas.strip.split(/ /).map{|x| "#{x}:#{config(:mongo_port)}"}
+                   repl_arr << "#{config(:mongo_host)}:#{config(:mongo_port)}"
+                   Mongo::ReplSetConnection.new(repl_arr, :read => :secondary)\
+                                           .db(config(:mongo_db))
+                 end
+        init_db(@mongo) if @mongo.collections.size <= 0
+        @mongo
+      else
+        @mongo
       end
-      col
     end
 
     # Declare an index on +field+ for +collection+ if it does not exist
     def ensure_index(collection, field)
-      col = @enttodb[collection]
+      col = get_entity(collection)
 
       exists = col.index_information.find {|k,v|
         k == "#{field}_1"
@@ -115,21 +151,51 @@ module GHTorrent
       end
     end
 
-  end
-end
+    def init_db(mongo)
+      ENTITIES.each {|x| mongo.collection(x.to_s)}
+
+      # Ensure that the necessary indexes exist
+      ensure_index(:events, "id")
+      ensure_index(:users, "login")
+      ensure_index(:commits, "sha")
+      ensure_index(:repos, "name")
+      ensure_index(:followers, "follows")
+      ensure_index(:org_members, "org")
+      ensure_index(:commit_comments, "repo")
+      ensure_index(:commit_comments, "user")
+      ensure_index(:commit_comments, "commit_id")
+      ensure_index(:repo_collaborators, "repo")
+      ensure_index(:repo_collaborators, "owner")
+      ensure_index(:repo_collaborators, "login")
+      ensure_index(:watchers, "repo")
+      ensure_index(:watchers, "owner")
+      ensure_index(:watchers, "login")
+      ensure_index(:pull_requests, "repo")
+      ensure_index(:pull_requests, "owner")
+      ensure_index(:forks, "repo")
+      ensure_index(:forks, "owner")
+      ensure_index(:forks, "id")
+      ensure_index(:issue_comments, "repo")
+      ensure_index(:issue_comments, "owner")
+      ensure_index(:issue_comments, "issue_id")
+      ensure_index(:issue_comments, "id")
+      ensure_index(:pull_request_comments, "repo")
+      ensure_index(:pull_request_comments, "owner")
+      ensure_index(:pull_request_comments, "pullreq_id")
+      ensure_index(:pull_request_comments, "id")
+    end
 
-class BSON::OrderedHash
-
-  # Convert a BSON result to a +Hash+
-  def to_h
-    inject({}) do |acc, element|
-      k, v = element;
-      acc[k] = if v.class == BSON::OrderedHash then
-                 v.to_h
-               else
-                 v
-               end;
-      acc
+    def rescue_connection_failure(max_retries=60)
+      retries = 0
+      begin
+        yield
+      rescue Mongo::ConnectionFailure => ex
+        retries += 1
+        raise ex if retries > max_retries
+        sleep(0.5)
+        @mongo.refresh if @mongo.class == Mongo::ReplSetConnection
+        retry
+      end
     end
   end
-end
+end