ghtorrent 0.2 → 0.3

data/README.md

@@ -1,4 +1,4 @@
-github-mirror: Mirror and process the Github event steam
+ghtorrent: Mirror and process the Github event steam
 =========================================================
 
 A collection of scripts used to mirror the Github event stream, for 
@@ -9,8 +9,8 @@ GHTorrent relies on the following software to work:
 
 * MongoDB > 2.0
 * RabbitMQ >= 2.7
-* An SQL database compatible with [Sequel](http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html). GHTorrent is tested with SQLite and MySQL,
-so your mileage may vary if you are using other databases.
+* An SQL database compatible with [Sequel](http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html). 
+GHTorrent is tested with SQLite and MySQL, so your mileage may vary if you are using other databases.
 
 GHTorrent is written in Ruby (tested with 1.8 and JRuby). To install 
 it as a Gem do:
@@ -19,6 +19,14 @@ it as a Gem do:
 sudo gem install ghtorrent 
 </code>
 
+Depending on which SQL database you want to use, install the appropriate dependency gem.
+GHTorrent already installs the `sqlite3` gem (if it fails, install the development
+package for `sqlite3` for your system).
+
+<code>
+sudo gem install mysql2 #or postgres
+</code>
+
 #### Configuring
 
 Copy the contents of the 
@@ -124,7 +132,7 @@ please consider citing the following paper:
 
 Georgios Gousios <gousiosg@gmail.com>
 
-Diomidis Spinellis
+[Diomidis Spinellis](http://www.dmst.aueb.gr/dds) <dds@aueb.gr>
 
 #### License
 

data/bin/ght-data-retrieval

@@ -38,7 +38,7 @@ class GHTDataRetrieval < GHTorrent::Command
 
   include GHTorrent::Settings
 
-  attr_reader :settings
+  attr_reader :settings, :name
 
   def parse(msg)
     JSON.parse(msg)
@@ -71,13 +71,23 @@ class GHTDataRetrieval < GHTorrent::Command
     %w(PushEvent WatchEvent FollowEvent)
   end
 
+  def prepare_options(options)
+      @name = "ght-data-retrieval"
+  end
+
   def go
     @gh = GHTorrent::Mirror.new(options[:config])
     @settings = @gh.settings
 
     # Graceful exit
-    Signal.trap('INT') { AMQP.stop { EM.stop } }
-    Signal.trap('TERM') { AMQP.stop { EM.stop } }
+    Signal.trap('INT') {
+      info ("Received SIGINT, exiting")
+      AMQP.stop { EM.stop }
+    }
+    Signal.trap('TERM') {
+      info ("Received SIGTERM, exiting")
+      AMQP.stop { EM.stop }
+    }
 
     AMQP.start(:host => config(:amqp_host),
                :port => config(:amqp_port),

data/bin/ght-load

@@ -113,7 +113,7 @@ Loads object ids from a collection to a queue for further processing.
                      :commits
                  end
 
-    puts "Loading form collection #{collection}"
+    puts "Loading from collection #{collection}"
     puts "Loading items after #{Time.at(options[:earliest])}" if options[:verbose]
 
     what = case

data/bin/ght-mirror-events

@@ -41,26 +41,51 @@ class GHTMirrorEvents < GHTorrent::Command
   include GHTorrent::Settings
   include GHTorrent::Logging
   include GHTorrent::Persister
+  include GHTorrent::APIClient
 
   attr_reader :settings
 
-  # Retrieve events from Github, store them in the DB
-  def retrieve(exchange)
-    begin
-      new = dupl = 0
-      events = @gh.get_events
-
-      events.each do |e|
-        unless @persister.find(:events, {'id' => e['id']}).empty?
-          info "Already got #{e['id']}"
-          dupl += 1
-          next
-        end
+  def initialize(args)
+    super(args)
+    @args = args
+    @name = self.class.name
+  end
 
+  def store_count(events)
+    stored = Array.new
+    new = dupl = 0
+    events.each do |e|
+      if @persister.find(:events, {'id' => e['id']}).empty?
+        stored << e
         new += 1
         @persister.store(:events, e)
         info "Added #{e['id']}"
+      else
+        info "Already got #{e['id']}"
+        dupl += 1
+      end
+    end
+    return new, dupl, stored
+  end
 
+  # Retrieve events from Github, store them in the DB
+  def retrieve(exchange)
+    begin
+      new = dupl = 0
+      events = api_request "https://api.github.com/events"
+      (new, dupl, stored) = store_count events
+
+      # This means that first page cannot contain all new events. Go
+      # up to 10 pages back to find all new events not contained in first page.
+      if dupl == 0
+        events = paged_api_request "https://api.github.com/events", 10
+        (new1, dupl1, stored1) = store_count events
+        stored = stored | stored1
+        new = new + new1
+        new
+      end
+
+      stored.each do |e|
         msg = JSON.dump(e)
         key = "evt.%s" % e['type']
         exchange.publish msg, :persistent => true, :routing_key => key
@@ -73,19 +98,24 @@ class GHTMirrorEvents < GHTorrent::Command
   end
 
   def prepare_options(options)
-
+    @name = "ght-mirror-events"
   end
 
   def go
-
     @gh = GHTorrent::Mirror.new(options[:config])
     @settings = @gh.settings
     @persister = connect(:mongo, @settings)
     @logger = Logger.new(STDOUT)
 
     # Graceful exit
-    Signal.trap('INT') { AMQP.stop { EM.stop } }
-    Signal.trap('TERM') { AMQP.stop { EM.stop } }
+    Signal.trap('INT') {
+      info ("Received SIGINT, exiting")
+      AMQP.stop { EM.stop }
+    }
+    Signal.trap('TERM') {
+      info ("Received SIGTERM, exiting")
+      AMQP.stop { EM.stop }
+    }
 
     # The event loop
     AMQP.start(:host => config(:amqp_host),
@@ -105,7 +135,7 @@ class GHTMirrorEvents < GHTorrent::Command
       # Initial delay for the retrieve event loop
       retrieval_delay = config(:mirror_pollevery)
 
-      # Retrieve commits.
+      # Retrieve events
       retriever = EventMachine.add_periodic_timer(retrieval_delay) do
         (new, dupl) = retrieve exchange
         dupl_msgs += dupl

data/bin/ght-periodic-dump

@@ -6,23 +6,53 @@
 # Directory to place compressed files and torrents
 OUTDIR=/home/data/github-mirror/dumps
 
-# Base URL for HTTP dir containing torrents and data 
+# Base URL for HTTP dir containing torrents and data
 WEBSEED=http://ikaria.dmst.aueb.gr/ghtorrent/
 
+usage()
+{
+	echo "Usage: $0 [-f 'yyyy-mm-dd hh:mm'] [-t 'yyyy-mm-dd hh:mm']"
+	echo "          [-c collection_to_dump]"
+	echo "Dump the database. -f earliest record timestamp"
+	echo "                   -t latest record timestamp"
+	echo "                   -c collection to dump (default: all)"
+}
+
+if [ -z $1 ]
+then
+	usage
+	exit 1
+fi
+
+while getopts "f:t:c:" o
+do
+	case $o in
+	f) 	timeStart=`date -d "$OPTARG" +%s` ;;
+	t) 	timeEnd=`date -d "$OPTARG" +%s` ;;
+	c) 	collection=$OPTARG ;;
+	\?)     echo "Invalid option: -$OPTARG" >&2
+		usage
+	        exit 1
+		;;
+	esac
+done
+
+
 # Time to start dumping from
-if [ -r lastrun ]
+if [ -z $timeStart ]
 then
-	timeStart=`cat lastrun`
-else
-	timeStart=0
+	if [ -r lastrun ]
+	then
+		timeStart=`cat lastrun`
+	else
+		timeStart=0
+	fi
 fi
 
 # Time to end dumping
-if [ "$1" = "" ]
+if [ -z $timeEnd ]
 then
 	timeEnd=`date +%s`
-else
-	timeEnd=`date -d "$1" +%s` || exit 1
 fi
 
 # Name used for the files
@@ -37,11 +67,19 @@ dateName=`date -d @$timeEnd -u +'%Y-%m-%d'`
 endId=`printf '%08x0000000000000000' $timeEnd`
 startId=`printf '%08x0000000000000000' $timeStart`
 
+
+if [ -z $collection ]
+then
+	collections=`echo "show collections"|mongo --quiet github|egrep -v "system|bye"`
+else
+	collections=$collection
+fi
+
 echo "Dumping database from `date -d @$timeStart` to `date -d @$timeEnd`"
 
-collections=`echo "show collections"|mongo --quiet github|egrep -v "system|bye"`
-	
 rm -rf dump
+mkdir -p dump/github
+
 for col in $collections; do
 
 	echo "Dumping $col"
@@ -62,8 +100,8 @@ for col in $collections; do
 (
 	echo "Start date: `date -u -d @$timeStart +'%Y-%m-%dT%H:%M:%SZ'`"
 	echo "End date: `date -u -d @$timeEnd +'%Y-%m-%dT%H:%M:%SZ'`"
-	meta $col 
-) 
+	meta $col
+)
 done |
 tee README.$dateName.txt >dump/github/README.txt || exit 1
 
@@ -88,5 +126,5 @@ done
 echo $timeEnd >lastrun || exit 1
 
 # Clean up
-rm -rf dump 
+rm -rf dump
 

data/lib/ghtorrent.rb

@@ -1,7 +1,7 @@
 #require 'ghtorrent-old/ghtorrent-old'
 
 module GHTorrent
-  VERSION = 0.2
+  VERSION = 0.3
 end
 
 require 'ghtorrent/command'

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -30,8 +30,9 @@ module GHTorrent
 
   class BaseAdapter
 
-    ENTITIES = [:users, :commits, :followers, :repos, :events]
-
+    ENTITIES = [:users, :commits, :followers, :repos, :events, :org_members,
+        :commit_comments
+    ]
 
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.
     def store(entity, data = {})
@@ -87,5 +88,13 @@ module GHTorrent
         throw GHTorrentException.new("Perister: Entity #{entity} not known")
       end
     end
+
+    # Count the number of entries returned by +query+ without retrieving them.
+    # The +query+ can be any query supported by +find+.
+    def count(entity, query = {})
+      unless ENTITIES.include?(entity)
+        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+      end
+    end
   end
 end

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -49,6 +49,7 @@ module GHTorrent
 
     # Creates a new instance of the MongoDB persistence adapter.
     # Expects a parsed YAML settings document as input.
+    # Will create indexes on fields most frequently used in queries.
     def initialize(set)
       merge LOCALCONFIG
 
@@ -62,32 +63,31 @@ module GHTorrent
           :commits => get_collection("commits"),
           :repos => get_collection("repos"),
           :followers => get_collection("followers"),
-          :events => get_collection("events")
+          :events => get_collection("events"),
+          :org_members => get_collection("org_members"),
+          :commit_comments => get_collection("commit_comments")
       }
+
+      # Ensure that the necessary indexes exist
+      ensure_index(:users, "login")
+      ensure_index(:commits, "sha")
+      ensure_index(:repos, "name")
+      ensure_index(:followers, "follows")
+      ensure_index(:org_members, "org")
+      ensure_index(:commit_comments, "repo")
+      ensure_index(:commit_comments, "user")
+      ensure_index(:commit_comments, "commit_id")
     end
 
 
     def store(entity, data = {})
       super
-      col = @enttodb[entity]
-
-      if col.nil?
-        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
-      end
-
-      col.insert(data).to_s
+      get_entity(entity).insert(data).to_s
     end
 
     def find(entity, query = {})
       super
-
-      col = @enttodb[entity]
-
-      if col.nil?
-        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
-      end
-
-      result = col.find(query)
+      result = get_entity(entity).find(query)
       result.to_a.map { |r|
         r[@uniq] = r['_id'].to_s;
         r.to_h
@@ -100,12 +100,41 @@ module GHTorrent
       raise NotImplementedError
     end
 
+    # Count the number of items returned by +query+
+    def count(entity, query)
+      super
+      get_entity(entity).count(:query => query)
+    end
+
     private
 
     def get_collection(col)
       @mongo.collection(col.to_s)
     end
 
+    def get_entity(entity)
+      col = @enttodb[entity]
+
+      if col.nil?
+        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
+      end
+      col
+    end
+
+    # Declare an index on +field+ for +collection+ if it does not exist
+    def ensure_index(collection, field)
+      col = @enttodb[collection]
+
+      exists = col.index_information.find {|k,v|
+        k == "#{field}_1"
+      }
+
+      if exists.nil?
+        col.create_index(field, :background => true)
+        STDERR.puts "Creating index on #{collection}(#{field})"
+      end
+    end
+
   end
 end
 

data/lib/ghtorrent/api_client.rb

@@ -40,33 +40,67 @@ module GHTorrent
       @num_api_calls = 0
       @ts = Time.now().tv_sec()
     end
-    
+
+    # A paged request. Used when the result can expand to more than one
+    # result pages.
     def paged_api_request(url, pages = -1)
 
-      pg = if pages == -1 then
-             1000000
-           else
-             pages
-           end
-      result = Array.new
+      data = api_request_raw(url)
+
+      return [] if data.nil?
+
+      unless data.meta['link'].nil?
+        links = parse_links(data.meta['link'])
+
+        if pages > 0
+          pages = pages - 1
+          if pages == 0
+            return parse_request_result(data)
+          end
+        end
 
-      (1..pg).each { |x|
-        data = api_request("#{url}?page=#{x}")
-        result += data
-        break if data.empty?
-      }
-      result
+        if links['next'].nil?
+          parse_request_result(data)
+        else
+          parse_request_result(data) | paged_api_request(links['next'], pages)
+        end
+      else
+        parse_request_result(data)
+      end
     end
 
+    # A normal request. Returns a hash or an array of hashes representing the
+    # parsed JSON result.
     def api_request(url)
-      result = api_request_raw(url)
+      parse_request_result api_request_raw(url)
+    end
+
+    private
+
+    # Parse a Github link header
+    def parse_links(links)
+      links.split(/,/).reduce({}) do |acc, x|
+        matches = x.strip.match(/<(.*)>; rel=\"(.*)\"/)
+        acc[matches[2]] = matches[1]
+        acc
+      end
+    end
+
+    # Parse the JSON result array
+    def parse_request_result(result)
       if result.nil?
-        nil
+        []
       else
-        JSON.parse(result)
+        json = result.read
+        if json.nil?
+          []
+        else
+          JSON.parse(json)
+        end
       end
     end
 
+    # Do the actual request and return the result object
     def api_request_raw(url)
       #Rate limiting to avoid error requests
       if Time.now().tv_sec() - @ts < 60 then
@@ -86,7 +120,7 @@ module GHTorrent
       @num_api_calls += 1
       debug "APIClient: Request: #{url} (num_calls = #{@num_api_calls})"
       begin
-        open(url).read
+        open(url)
       rescue OpenURI::HTTPError => e
         case e.io.status[0].to_i
           # The following indicate valid Github return codes