ghtorrent 0.5 → 0.6

data/lib/ghtorrent/commands/ght_get_more_commits.rb

@@ -0,0 +1,116 @@
+require 'rubygems'
+require 'time'
+
+require 'ghtorrent/ghtorrent'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/command'
+require 'ghtorrent/retriever'
+
+class GHTMoreCommitsRetriever < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Retriever
+  include GHTorrent::Persister
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Retrieves more commits for the provided repository
+
+#{command_name} [options] owner repo
+
+#{command_name} options:
+    BANNER
+
+    options.opt :num, 'Number of commits to retrieve',
+                :short => 'n', :default => -1, :type => :int
+    options.opt :full, 'Retrieve all commits, filling in potential holes',
+                :short => 'f', :default => -1, :type => :int
+  end
+
+  def validate
+    super
+    Trollop::die "Two arguments are required" unless args[0] && !args[0].empty?
+
+    Trollop::die "-a and -n cannot be defined at the same time" \
+      if not options[:all].nil? and not options[:foo].nil?
+  end
+
+  def logger
+    @ght.logger
+  end
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def ext_uniq
+    @ext_uniq ||= config(:uniq_id)
+    @ext_uniq
+  end
+
+  def go
+
+    @ght ||= GHTorrent::Mirror.new(settings)
+    user_entry = @ght.transaction{@ght.ensure_user(ARGV[0], false, false)}
+
+    if user_entry.nil?
+      Trollop::die "Cannot find user #{owner}"
+    end
+
+    user = user_entry[:login]
+
+    repo_entry = @ght.transaction{@ght.ensure_repo(ARGV[0], ARGV[1], false, false, false)}
+
+    if repo_entry.nil?
+      Trollop::die "Cannot find repository #{owner}/#{ARGV[1]}"
+    end
+
+    repo = repo_entry[:name]
+    num_pages = if options[:num] == -1 then 1024 * 1024 else options[:n]/30 end
+    num_pages = if options[:full] == -1 then num_pages else 1024 * 1024 end
+    page = 0
+
+
+    head = unless options[:full] == -1
+             @ght.get_db.from(:commits).\
+                      where(:commits__project_id => repo_entry[:id]).\
+                      order(:created_at).\
+                      first.\
+                      select(:sha)
+           else
+             "master"
+           end
+
+    total_commits = 0
+    while (page < num_pages)
+      begin
+        logger.debug("Retrieving more commits for #{user}/#{repo} from head: #{head}")
+
+        commits = retrieve_commits(repo, head, user, 1)
+        page += 1
+        if commits.nil? or commits.empty? or commits.size == 1
+          page = num_pages # To break the loop
+          break
+        end
+
+        total_commits += commits.size
+        head = commits.last['sha']
+
+        commits.map do |c|
+          @ght.transaction do
+            @ght.ensure_commit(repo, c['sha'], user)
+          end
+        end
+      rescue Exception => e
+        logger.warn("Error processing: #{e}")
+        logger.warn(e.backtrace.join("\n"))
+      end
+    end
+    logger.debug("Processed #{total_commits} commits for #{user}/#{repo}")
+  end
+end
+
+
+#vim: set filetype=ruby expandtab tabstop=2 shiftwidth=2 autoindent smartindent:

data/lib/ghtorrent/commands/ght_load.rb

@@ -0,0 +1,227 @@
+require 'rubygems'
+require 'mongo'
+require 'amqp'
+require 'set'
+require 'eventmachine'
+require 'pp'
+require "amqp/extensions/rabbitmq"
+
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/persister'
+require 'ghtorrent/command'
+require 'ghtorrent/bson_orderedhash'
+
+class GHTLoad < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Persister
+
+  def col_info()
+    {
+        :commits => {
+            :name => "commits",
+            :payload => "commit.id",
+            :unq => "commit.id",
+            :col => persister.get_underlying_connection.collection(:commits.to_s),
+            :routekey => "commit.%s"
+        },
+        :events => {
+            :name => "events",
+            :payload => "",
+            :unq => "type",
+            :col => persister.get_underlying_connection.collection(:events.to_s),
+            :routekey => "evt.%s"
+        }
+    }
+  end
+
+  def persister
+    @persister ||= connect(:mongo, @settings)
+    @persister
+  end
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Loads object ids from a collection to a queue for further processing.
+
+#{command_name} [options] collection
+
+#{command_name} options:
+    BANNER
+
+    options.opt :earliest, 'Seconds since epoch of earliest item to load',
+                :short => 'e', :default => 0, :type => :int
+    options.opt :number, 'Number of items to load (-1 means all)',
+                :short => 'n', :type => :int, :default => -1
+    options.opt :filter,
+                'Filter items by regexp on item attributes: item.attr=regexp',
+                :short => 'f', :type => String, :multi => true
+  end
+
+  def validate
+    super
+    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
+    filter = options[:filter]
+    case
+      when filter.is_a?(Array)
+        options[:filter].each { |x|
+          Trollop::die "not a valid filter #{x}" unless is_filter_valid?(x)
+        }
+      when filter == []
+        # Noop
+      else
+        Trollop::die "A filter can only be a string"
+    end
+  end
+
+  def go
+    # Message tags await publisher ack
+    awaiting_ack = SortedSet.new
+
+    # Num events read
+    num_read = 0
+
+    collection = case args[0]
+                   when "events"
+                     :events
+                   when "commits"
+                     :commits
+                 end
+
+    puts "Loading from collection #{collection}"
+    puts "Loading items after #{Time.at(options[:earliest])}" if options[:verbose]
+    puts "Loading #{options[:number]} items" if options[:verbose] && options[:number] != -1
+
+    what = case
+             when options[:filter].is_a?(Array)
+               options[:filter].reduce({}) { |acc,x|
+                 (k,r) = x.split(/=/)
+                 acc[k] = Regexp.new(r)
+                 acc
+               }
+             when filter == []
+               {}
+           end
+
+    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
+
+    (puts "Mongo filter:"; pp what.merge(from)) if options[:verbose]
+
+    AMQP.start(:host => config(:amqp_host),
+               :port => config(:amqp_port),
+               :username => config(:amqp_username),
+               :password => config(:amqp_password)) do |connection|
+
+      channel = AMQP::Channel.new(connection)
+      exchange = channel.topic(config(:amqp_exchange),
+                               :durable => true, :auto_delete => false)
+
+      # What to do when the user hits Ctrl+c
+      show_stopper = Proc.new {
+        connection.close { EventMachine.stop }
+      }
+
+      # Read next 1000 items and queue them
+      read_and_publish = Proc.new {
+
+        to_read = if options.number == -1
+                    1000
+                  else
+                    if options.number - num_read - 1 <= 0
+                      -1
+                    else
+                      options.number - num_read - 1
+                    end
+                  end
+
+        read = 0
+        col_info[collection][:col].find(what.merge(from),
+                                        :skip => num_read,
+                                        :limit => to_read).each do |e|
+
+          payload = read_value(e, col_info[collection][:payload])
+          payload = if payload.class == BSON::OrderedHash
+                      payload.delete "_id" # Inserted by MongoDB on event insert
+                      payload.to_json
+                    end
+          read += 1
+          unq = read_value(e, col_info[collection][:unq])
+          if unq.class != String or unq.nil? then
+            throw Exception.new("Unique value can only be a String")
+          end
+
+          key = col_info[collection][:routekey] % unq
+
+          exchange.publish payload, :persistent => true, :routing_key => key
+
+          num_read += 1
+          puts("Publish id = #{payload[unq]} (#{num_read} total)") if options.verbose
+          awaiting_ack << num_read
+        end
+
+        # Nothing new in the DB and no msgs waiting ack
+        if (read == 0 and awaiting_ack.size == 0) or to_read == -1
+          puts("Finished reading, exiting")
+          show_stopper.call
+        end
+      }
+
+      # Remove acknowledged or failed msg tags from the queue
+      # Trigger more messages to be read when ack msg queue size drops to zero
+      publisher_event = Proc.new { |ack|
+        if ack.multiple then
+          awaiting_ack.delete_if { |x| x <= ack.delivery_tag }
+        else
+          awaiting_ack.delete ack.delivery_tag
+        end
+
+        if awaiting_ack.size == 0
+          puts("ACKS.size= #{awaiting_ack.size}") if options.verbose
+          EventMachine.next_tick do
+            read_and_publish.call
+          end
+        end
+      }
+
+      # Await publisher confirms
+      channel.confirm_select
+
+      # Callback when confirms have arrived
+      channel.on_ack do |ack|
+        puts "ACK: tag=#{ack.delivery_tag}, mul=#{ack.multiple}" if options.verbose
+        publisher_event.call(ack)
+      end
+
+      # Callback when confirms failed.
+      channel.on_nack do |nack|
+        puts "NACK: tag=#{nack.delivery_tag}, mul=#{nack.multiple}" if options.verbose
+        publisher_event.call(nack)
+      end
+
+      # Signal handlers
+      Signal.trap('INT', show_stopper)
+      Signal.trap('TERM', show_stopper)
+
+      # Trigger start processing
+      EventMachine.add_timer(0.1) do
+        read_and_publish.call
+      end
+    end
+  end
+
+  private
+
+  def is_filter_valid?(filter)
+    (k, r) = filter.split(/=/)
+    return false if r.nil?
+    begin
+      Regexp.new(r)
+      true
+    rescue
+      false
+    end
+  end
+end
+
+#vim: set filetype=ruby expandtab tabstop=2 shiftwidth=2 autoindent smartindent:

data/lib/ghtorrent/commands/ght_mirror_events.rb

@@ -0,0 +1,147 @@
+require 'rubygems'
+require 'yaml'
+require 'amqp'
+require 'eventmachine'
+require 'json'
+require 'logger'
+
+require 'ghtorrent/api_client'
+require 'ghtorrent/settings'
+require 'ghtorrent/logging'
+require 'ghtorrent/persister'
+require 'ghtorrent/command'
+
+class GHTMirrorEvents < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+  include GHTorrent::Persister
+  include GHTorrent::APIClient
+
+  def logger
+    @logger
+  end
+
+  def store_count(events)
+    stored = Array.new
+    new = dupl = 0
+    events.each do |e|
+      if @persister.find(:events, {'id' => e['id']}).empty?
+        stored << e
+        new += 1
+        @persister.store(:events, e)
+        info "Added #{e['id']}"
+      else
+        info "Already got #{e['id']}"
+        dupl += 1
+      end
+    end
+    return new, dupl, stored
+  end
+
+  # Retrieve events from Github, store them in the DB
+  def retrieve(exchange)
+    begin
+      new = dupl = 0
+      events = api_request "https://api.github.com/events", false
+      (new, dupl, stored) = store_count events
+
+      # This means that first page cannot contain all new events. Go
+      # up to 10 pages back to find all new events not contained in first page.
+      if dupl == 0
+        events = paged_api_request "https://api.github.com/events"
+        (new1, dupl1, stored1) = store_count events
+        stored = stored | stored1
+        new = new + new1
+        new
+      end
+
+      stored.each do |e|
+        msg = JSON.dump(e)
+        key = "evt.%s" % e['type']
+        exchange.publish msg, :persistent => true, :routing_key => key
+      end
+      return new, dupl
+    rescue Exception => e
+      STDERR.puts e.message
+      STDERR.puts e.backtrace
+    end
+  end
+
+  def go
+    @persister = connect(:mongo, @settings)
+    @logger = Logger.new(STDOUT)
+
+    # Graceful exit
+    Signal.trap('INT') {
+      info "Received SIGINT, exiting"
+      AMQP.stop { EM.stop }
+    }
+    Signal.trap('TERM') {
+      info "Received SIGTERM, exiting"
+      AMQP.stop { EM.stop }
+    }
+
+    # The event loop
+    AMQP.start(:host => config(:amqp_host),
+               :port => config(:amqp_port),
+               :username => config(:amqp_username),
+               :password => config(:amqp_password)) do |connection|
+
+      # Statistics used to recalibrate event delays
+      dupl_msgs = new_msgs = 1
+
+      debug "connected to rabbit"
+
+      channel = AMQP::Channel.new(connection)
+      exchange = channel.topic(config(:amqp_exchange), :durable => true,
+                               :auto_delete => false)
+
+      # Initial delay for the retrieve event loop
+      retrieval_delay = config(:mirror_pollevery)
+
+      # Retrieve events
+      retriever = EventMachine.add_periodic_timer(retrieval_delay) do
+        (new, dupl) = retrieve exchange
+        dupl_msgs += dupl
+        new_msgs += new
+      end
+
+      # Adjust event retrieval delay time to reduce load to Github
+      EventMachine.add_periodic_timer(120) do
+        ratio = (dupl_msgs.to_f / (dupl_msgs + new_msgs).to_f)
+
+        info("Stats: #{new_msgs} new, #{dupl_msgs} duplicate, ratio: #{ratio}")
+
+        new_delay = if ratio >= 0 and ratio < 0.3 then
+                      -1
+                    elsif ratio >= 0.3 and ratio <= 0.5 then
+                      0
+                    elsif ratio > 0.5 and ratio < 1 then
+                      +1
+                    end
+
+        # Reset counters for new loop
+        dupl_msgs = new_msgs = 0
+
+        # Update the retrieval delay and restart the event retriever
+        if new_delay != 0
+
+          # Stop the retriever task and adjust retrieval delay
+          retriever.cancel
+          retrieval_delay = retrieval_delay + new_delay
+          info("Setting event retrieval delay to #{retrieval_delay} secs")
+
+          # Restart the retriever
+          retriever = EventMachine.add_periodic_timer(retrieval_delay) do
+            (new, dupl) = retrieve exchange
+            dupl_msgs += dupl
+            new_msgs += new
+          end
+        end
+      end
+    end
+  end
+end
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :