ghtorrent 0.5 → 0.6

data/CHANGELOG

@@ -1,6 +1,21 @@
+= Version 0.6
+
+	* Support retrieval of issues, issue events and issue history
+	* Support for setting username/password for performing requests
+	* Respect by default Github's x-ratelimit-remaining header
+	* Selective processing of events for user-specified repos
+	* New tool (ght-get-more-commits) to retrieve all commits for a repository
+	* New tool (ght-process-events) to process just one event by id
+	* Retrieve 100 items at once by default on multipage requests
+	* Rename watchers -> stargazers, as per Github API change
+	* Fixes to bugs that permitted efficient processing of multipage requests
+	* Several fixes on how pull requests are being processed
+	* Users with invalid git setups are now allowed
+	* Compatibility with Ruby 1.8 restored
+
 = Version 0.5
 
-    * Generic methods for retrieving items that are bound to repositories
+  * Generic methods for retrieving items that are bound to repositories
 	* Processing of pull requests with commits, comments and history
 	* Processing of project forks
 	* New tool (ght-load) to filter and load events to the queue

data/README.md

@@ -129,9 +129,14 @@ please consider citing the following paper:
 
 > Georgios Gousios and Diomidis Spinellis, "GHTorrent: GitHub’s data from a firehose," in _MSR '12: Proceedings of the 9th Working Conference on Mining Software Repositories_, June 2-–3, 2012. Zurich, Switzerland.
 
+See also the following presentation:
+
+<iframe src="http://www.slideshare.net/slideshow/embed_code/13184524?rel=0" width="342" height="291" frameborder="0" marginwidth="0" marginheight="0" scrolling="no" style="border:1px solid #CCC;border-width:1px 1px 0;margin-bottom:5px" allowfullscreen/> 
+<div style="margin-bottom:5px"> <strong> <a href="http://www.slideshare.net/gousiosg/ghtorrent-githubs-data-from-a-firehose-13184524" title="GHTorrent: Github&#39;s Data from a Firehose" target="_blank">GHTorrent: Github&#39;s Data from a Firehose</a> </strong> </div>
+
 #### Authors
 
-Georgios Gousios <gousiosg@gmail.com>
+[Georgios Gousios](http://istlab.dmst.aueb.gr/~george) <gousiosg@gmail.com>
 
 [Diomidis Spinellis](http://www.dmst.aueb.gr/dds) <dds@aueb.gr>
 

data/bin/ght-data-retrieval

@@ -1,166 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require 'amqp'
-require 'json'
-require 'pp'
+require 'ghtorrent'
 
-require 'ghtorrent/ghtorrent'
-require 'ghtorrent/settings'
-require 'ghtorrent/logging'
-require 'ghtorrent/command'
-
-class GHTDataRetrieval < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Logging
-
-  def parse(msg)
-    JSON.parse(msg)
-  end
-
-  def PushEvent(data)
-    data['payload']['commits'].each do |c|
-      url = c['url'].split(/\//)
-
-      @gh.get_commit url[4], url[5], url[7]
-    end
-  end
-
-  def WatchEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    watcher = data['actor']['login']
-    created_at = data['created_at']
-
-    @gh.get_watcher owner, repo, watcher, created_at
-  end
-
-  def FollowEvent(data)
-    follower = data['actor']['login']
-    followed = data['payload']['target']['login']
-    created_at = data['created_at']
-
-    @gh.get_follower(follower, followed, created_at)
-  end
-
-  def MemberEvent(data)
-    owner = data['actor']['login']
-    repo = data['repo']['name'].split(/\//)[1]
-    new_member = data['payload']['member']['login']
-    created_at = data['created_at']
-
-    @gh.get_project_member(owner, repo, new_member, created_at)
-  end
-
-  def CommitCommentEvent(data)
-    user = data['actor']['login']
-    repo = data['repo']['name'].split(/\//)[1]
-    id = data['payload']['comment']['id']
-    created_at = data['created_at']
-
-    @gh.get_commit_comment(user, repo, id, created_at)
-  end
-
-  def PullRequestEvent(data)
-    owner = data['payload']['pull_request']['base']['repo']['owner']['login']
-    repo = data['payload']['pull_request']['base']['repo']['name']
-    pullreq_id = data['payload']['number']
-    action = data['payload']['action']
-    created_at = data['created_at']
-
-    @gh.get_pull_request(owner, repo, pullreq_id, action, created_at)
-  end
-
-  def ForkEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    fork_id = data['payload']['forkee']['id']
-    created_at = data['created_at']
-
-    @gh.get_fork(owner, repo, fork_id, created_at)
-  end
-
-  def PullRequestReviewCommentEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    comment_id = data['payload']['comment']['id']
-    pullreq_id = data['payload']['comment']['_links']['pull_request']['href'].split(/\//)[-1]
-    created_at = data['created_at']
-
-    @gh.get_pullreq_comment(owner, repo, pullreq_id, comment_id, created_at)
-  end
-
-  def IssueCommentEvent(data)
-    owner = data['repo']['name'].split(/\//)[0]
-    repo = data['repo']['name'].split(/\//)[1]
-    pullreq_id = data['payload']['forkee']['id']
-    created_at = data['created_at']
-
-    @gh.get_issue_comment(owner, repo, issue_id, comment_id, created_at)
-  end
-
-  def handlers
-    %w(PushEvent WatchEvent FollowEvent MemberEvent CommitCommentEvent PullRequestEvent ForkEvent PullRequestReviewCommentEvent)
-    #%w(PullRequestReviewCommentEvent)
-  end
-
-  def logger
-    @gh.logger
-  end
-
-  def go
-    @gh = GHTorrent::Mirror.new(@settings)
-
-    # Graceful exit
-    Signal.trap('INT') {
-      info "GHTDataRetrieval: Received SIGINT, exiting"
-      AMQP.stop { EM.stop }
-    }
-    Signal.trap('TERM') {
-      info "GHTDataRetrieval: Received SIGTERM, exiting"
-      AMQP.stop { EM.stop }
-    }
-
-    AMQP.start(:host => config(:amqp_host),
-               :port => config(:amqp_port),
-               :username => config(:amqp_username),
-               :password => config(:amqp_password)) do |connection|
-
-      channel = AMQP::Channel.new(connection, :prefetch => config(:amqp_prefetch))
-      exchange = channel.topic(config(:amqp_exchange), :durable => true,
-                               :auto_delete => false)
-
-      handlers.each { |h|
-        queue = channel.queue("#{h}s", {:durable => true})\
-                       .bind(exchange, :routing_key => "evt.#{h}")
-
-        info "GHTDataRetrieval: Binding handler #{h} to routing key evt.#{h}"
-
-        queue.subscribe(:ack => true) do |headers, msg|
-          begin
-            data = parse(msg)
-            info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
-	          send(h, data)
-            headers.ack
-            info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
-          rescue Exception => e
-            # Give a message a chance to be reprocessed
-            if headers.redelivered?
-              data = parse(msg)
-              warn "GHTDataRetrieval: Could not process event: #{data['type']}-#{data['id']}"
-              headers.reject(:requeue => false)
-            else
-              headers.reject(:requeue => true)
-            end
-
-            STDERR.puts e
-            STDERR.puts e.backtrace.join("\n")
-          end
-        end
-      }
-    end
-  end
-end
-
-GHTDataRetrieval.run
+GHTDataRetrieval.run(ARGV)

data/bin/ght-get-more-commits

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ghtorrent'
+
+GHTMoreCommitsRetriever.run(ARGV)

data/bin/ght-load

@@ -1,230 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require 'mongo'
-require 'amqp'
-require 'set'
-require 'eventmachine'
-require 'pp'
-require "amqp/extensions/rabbitmq"
-
-require 'ghtorrent/settings'
-require 'ghtorrent/logging'
-require 'ghtorrent/persister'
-require 'ghtorrent/command'
-require 'ghtorrent/bson_orderedhash'
-
-class GHTLoad < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Persister
-
-  def col_info()
-    {
-        :commits => {
-            :name => "commits",
-            :payload => "commit.id",
-            :unq => "commit.id",
-            :col => persister.get_underlying_connection.collection(:commits.to_s),
-            :routekey => "commit.%s"
-        },
-        :events => {
-            :name => "events",
-            :payload => "",
-            :unq => "type",
-            :col => persister.get_underlying_connection.collection(:events.to_s),
-            :routekey => "evt.%s"
-        }
-    }
-  end
-
-  def persister
-    @persister ||= connect(:mongo, @settings)
-    @persister
-  end
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Loads object ids from a collection to a queue for further processing.
-
-#{command_name} [options] collection
-
-#{command_name} options:
-    BANNER
-
-    options.opt :earliest, 'Seconds since epoch of earliest item to load',
-      :short => 'e', :default => 0, :type => :int
-    options.opt :number, 'Number of items to load (-1 means all)',
-                :short => 'n', :type => :int, :default => -1
-    options.opt :filter,
-                'Filter items by regexp on item attributes: item.attr=regexp',
-                :short => 'f', :type => String, :multi => true
-  end
-
-  def validate
-    super
-    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
-    filter = options[:filter]
-    case
-      when filter.is_a?(Array)
-        options[:filter].each { |x|
-          Trollop::die "not a valid filter #{x}" unless is_filter_valid?(x)
-        }
-      when filter == []
-        # Noop
-      else
-        Trollop::die "A filter can only be a string"
-    end
-  end
-
-  def go
-    # Message tags await publisher ack
-    awaiting_ack = SortedSet.new
-
-    # Num events read
-    num_read = 0
-
-    collection = case args[0]
-                   when "events"
-                     :events
-                   when "commits"
-                     :commits
-                 end
-
-    puts "Loading from collection #{collection}"
-    puts "Loading items after #{Time.at(options[:earliest])}" if options[:verbose]
-    puts "Loading #{options[:number]} items" if options[:verbose] && options[:number] != -1
-
-    what = case
-             when options[:filter].is_a?(Array)
-               options[:filter].reduce({}) { |acc,x|
-                 (k,r) = x.split(/=/)
-                 acc[k] = Regexp.new(r)
-                 acc
-               }
-             when filter == []
-               {}
-           end
-
-    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
-
-    (puts "Mongo filter:"; pp what.merge(from)) if options[:verbose]
-
-    AMQP.start(:host => config(:amqp_host),
-               :port => config(:amqp_port),
-               :username => config(:amqp_username),
-               :password => config(:amqp_password)) do |connection|
-
-      channel = AMQP::Channel.new(connection)
-      exchange = channel.topic(config(:amqp_exchange),
-                               :durable => true, :auto_delete => false)
-
-      # What to do when the user hits Ctrl+c
-      show_stopper = Proc.new {
-        connection.close { EventMachine.stop }
-      }
-
-      # Read next 1000 items and queue them
-      read_and_publish = Proc.new {
-
-        to_read = if options.number == -1
-                    1000
-                  else
-                    if options.number - num_read - 1 <= 0
-                      -1
-                    else
-                      options.number - num_read - 1
-                    end
-                  end
-
-        read = 0
-        col_info[collection][:col].find(what.merge(from),
-                                       :skip => num_read,
-                                       :limit => to_read).each do |e|
-
-          payload = read_value(e, col_info[collection][:payload])
-          payload = if payload.class == BSON::OrderedHash
-                      payload.delete "_id" # Inserted by MongoDB on event insert
-                      payload.to_json
-                    end
-          read += 1
-          unq = read_value(e, col_info[collection][:unq])
-          if unq.class != String or unq.nil? then
-            throw Exception("Unique value can only be a String")
-          end
-
-          key = col_info[collection][:routekey] % unq
-
-          exchange.publish payload, :persistent => true, :routing_key => key
-
-          num_read += 1
-          puts("Publish id = #{payload[unq]} (#{num_read} total)") if options.verbose
-          awaiting_ack << num_read
-        end
-
-        # Nothing new in the DB and no msgs waiting ack
-        if (read == 0 and awaiting_ack.size == 0) or to_read == -1
-          puts("Finished reading, exiting")
-          show_stopper.call
-        end
-      }
-
-      # Remove acknowledged or failed msg tags from the queue
-      # Trigger more messages to be read when ack msg queue size drops to zero
-      publisher_event = Proc.new { |ack|
-        if ack.multiple then
-          awaiting_ack.delete_if { |x| x <= ack.delivery_tag }
-        else
-          awaiting_ack.delete ack.delivery_tag
-        end
-
-        if awaiting_ack.size == 0
-          puts("ACKS.size= #{awaiting_ack.size}") if options.verbose
-          EventMachine.next_tick do
-            read_and_publish.call
-          end
-        end
-      }
-
-      # Await publisher confirms
-      channel.confirm_select
-
-      # Callback when confirms have arrived
-      channel.on_ack do |ack|
-        puts "ACK: tag=#{ack.delivery_tag}, mul=#{ack.multiple}" if options.verbose
-        publisher_event.call(ack)
-      end
-
-      # Callback when confirms failed.
-      channel.on_nack do |nack|
-        puts "NACK: tag=#{nack.delivery_tag}, mul=#{nack.multiple}" if options.verbose
-        publisher_event.call(nack)
-      end
-
-      # Signal handlers
-      Signal.trap('INT', show_stopper)
-      Signal.trap('TERM', show_stopper)
-
-      # Trigger start processing
-      EventMachine.add_timer(0.1) do
-        read_and_publish.call
-      end
-    end
-  end
-
-  private
-
-  def is_filter_valid?(filter)
-    (k, r) = filter.split(/=/)
-    return false if r.nil?
-    begin
-      Regexp.new(r)
-      true
-    rescue
-      false
-    end
-  end
-end
+require 'ghtorrent'
 
 GHTLoad.run
 

data/bin/ght-mirror-events

@@ -1,151 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require 'yaml'
-require 'amqp'
-require 'eventmachine'
-require 'json'
-require 'logger'
+require 'ghtorrent'
 
-require 'ghtorrent/api_client'
-require 'ghtorrent/settings'
-require 'ghtorrent/logging'
-require 'ghtorrent/persister'
-require 'ghtorrent/command'
-
-class GHTMirrorEvents < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Logging
-  include GHTorrent::Persister
-  include GHTorrent::APIClient
-
-  def logger
-    @logger
-  end
-
-  def store_count(events)
-    stored = Array.new
-    new = dupl = 0
-    events.each do |e|
-      if @persister.find(:events, {'id' => e['id']}).empty?
-        stored << e
-        new += 1
-        @persister.store(:events, e)
-        info "Added #{e['id']}"
-      else
-        info "Already got #{e['id']}"
-        dupl += 1
-      end
-    end
-    return new, dupl, stored
-  end
-
-  # Retrieve events from Github, store them in the DB
-  def retrieve(exchange)
-    begin
-      new = dupl = 0
-      events = api_request "https://api.github.com/events", false
-      (new, dupl, stored) = store_count events
-
-      # This means that first page cannot contain all new events. Go
-      # up to 10 pages back to find all new events not contained in first page.
-      if dupl == 0
-        events = paged_api_request "https://api.github.com/events", 10
-        (new1, dupl1, stored1) = store_count events
-        stored = stored | stored1
-        new = new + new1
-        new
-      end
-
-      stored.each do |e|
-        msg = JSON.dump(e)
-        key = "evt.%s" % e['type']
-        exchange.publish msg, :persistent => true, :routing_key => key
-      end
-      return new, dupl
-    rescue Exception => e
-      STDERR.puts e.message
-      STDERR.puts e.backtrace
-    end
-  end
-
-  def go
-    @persister = connect(:mongo, @settings)
-    @logger = Logger.new(STDOUT)
-
-    # Graceful exit
-    Signal.trap('INT') {
-      info "Received SIGINT, exiting"
-      AMQP.stop { EM.stop }
-    }
-    Signal.trap('TERM') {
-      info "Received SIGTERM, exiting"
-      AMQP.stop { EM.stop }
-    }
-
-    # The event loop
-    AMQP.start(:host => config(:amqp_host),
-               :port => config(:amqp_port),
-               :username => config(:amqp_username),
-               :password => config(:amqp_password)) do |connection|
-
-      # Statistics used to recalibrate event delays
-      dupl_msgs = new_msgs = 1
-
-      debug "connected to rabbit"
-
-      channel = AMQP::Channel.new(connection)
-      exchange = channel.topic(config(:amqp_exchange), :durable => true,
-                               :auto_delete => false)
-
-      # Initial delay for the retrieve event loop
-      retrieval_delay = config(:mirror_pollevery)
-
-      # Retrieve events
-      retriever = EventMachine.add_periodic_timer(retrieval_delay) do
-        (new, dupl) = retrieve exchange
-        dupl_msgs += dupl
-        new_msgs += new
-      end
-
-      # Adjust event retrieval delay time to reduce load to Github
-      EventMachine.add_periodic_timer(120) do
-        ratio = (dupl_msgs.to_f / (dupl_msgs + new_msgs).to_f)
-
-        info("Stats: #{new_msgs} new, #{dupl_msgs} duplicate, ratio: #{ratio}")
-
-        new_delay = if ratio >= 0 and ratio < 0.3 then
-                      -1
-                    elsif ratio >= 0.3 and ratio <= 0.5 then
-                      0
-                    elsif ratio > 0.5 and ratio < 1 then
-                      +1
-                    end
-
-        # Reset counters for new loop
-        dupl_msgs = new_msgs = 0
-
-        # Update the retrieval delay and restart the event retriever
-        if new_delay != 0
-
-          # Stop the retriever task and adjust retrieval delay
-          retriever.cancel
-          retrieval_delay = retrieval_delay + new_delay
-          info("Setting event retrieval delay to #{retrieval_delay} secs")
-
-          # Restart the retriever
-          retriever = EventMachine.add_periodic_timer(retrieval_delay) do
-            (new, dupl) = retrieve exchange
-            dupl_msgs += dupl
-            new_msgs += new
-          end
-        end
-      end
-    end
-  end
-end
-
-GHTMirrorEvents.run
-
-# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :
+GHTMirrorEvents.run(ARGV)