ghtorrent 0.2

data/bin/ght-mirror-events

@@ -0,0 +1,154 @@
+#!/usr/bin/env ruby
+#
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'rubygems'
+require 'yaml'
+require 'amqp'
+require 'eventmachine'
+require 'ghtorrent'
+require 'json'
+require 'logger'
+
+class GHTMirrorEvents < GHTorrent::Command
+
+  include GHTorrent::Settings
+  include GHTorrent::Logging
+  include GHTorrent::Persister
+
+  attr_reader :settings
+
+  # Retrieve events from Github, store them in the DB
+  def retrieve(exchange)
+    begin
+      new = dupl = 0
+      events = @gh.get_events
+
+      events.each do |e|
+        unless @persister.find(:events, {'id' => e['id']}).empty?
+          info "Already got #{e['id']}"
+          dupl += 1
+          next
+        end
+
+        new += 1
+        @persister.store(:events, e)
+        info "Added #{e['id']}"
+
+        msg = JSON.dump(e)
+        key = "evt.%s" % e['type']
+        exchange.publish msg, :persistent => true, :routing_key => key
+      end
+      return new, dupl
+    rescue Exception => e
+      STDERR.puts e.message
+      STDERR.puts e.backtrace
+    end
+  end
+
+  def prepare_options(options)
+
+  end
+
+  def go
+
+    @gh = GHTorrent::Mirror.new(options[:config])
+    @settings = @gh.settings
+    @persister = connect(:mongo, @settings)
+    @logger = Logger.new(STDOUT)
+
+    # Graceful exit
+    Signal.trap('INT') { AMQP.stop { EM.stop } }
+    Signal.trap('TERM') { AMQP.stop { EM.stop } }
+
+    # The event loop
+    AMQP.start(:host => config(:amqp_host),
+               :port => config(:amqp_port),
+               :username => config(:amqp_username),
+               :password => config(:amqp_password)) do |connection|
+
+      # Statistics used to recalibrate event delays
+      dupl_msgs = new_msgs = 1
+
+      debug "connected to rabbit"
+
+      channel = AMQP::Channel.new(connection)
+      exchange = channel.topic(config(:amqp_exchange), :durable => true,
+                               :auto_delete => false)
+
+      # Initial delay for the retrieve event loop
+      retrieval_delay = config(:mirror_pollevery)
+
+      # Retrieve commits.
+      retriever = EventMachine.add_periodic_timer(retrieval_delay) do
+        (new, dupl) = retrieve exchange
+        dupl_msgs += dupl
+        new_msgs += new
+      end
+
+      # Adjust event retrieval delay time to reduce load to Github
+      EventMachine.add_periodic_timer(120) do
+        ratio = (dupl_msgs.to_f / (dupl_msgs + new_msgs).to_f)
+
+        info("Stats: #{new_msgs} new, #{dupl_msgs} duplicate, ratio: #{ratio}")
+
+        new_delay = if ratio >= 0 and ratio < 0.3 then
+                      -1
+                    elsif ratio >= 0.3 and ratio <= 0.5 then
+                      0
+                    elsif ratio > 0.5 and ratio < 1 then
+                      +1
+                    end
+
+        # Reset counters for new loop
+        dupl_msgs = new_msgs = 0
+
+        # Update the retrieval delay and restart the event retriever
+        if new_delay != 0
+
+          # Stop the retriever task and adjust retrieval delay
+          retriever.cancel
+          retrieval_delay = retrieval_delay + new_delay
+          info("Setting event retrieval delay to #{retrieval_delay} secs")
+
+          # Restart the retriever
+          retriever = EventMachine.add_periodic_timer(retrieval_delay) do
+            (new, dupl) = retrieve exchange
+            dupl_msgs += dupl
+            new_msgs += new
+          end
+        end
+      end
+    end
+  end
+end
+
+GHTMirrorEvents.run
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/bin/ght-periodic-dump

@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Create the periodic database dump files
+#
+
+# Directory to place compressed files and torrents
+OUTDIR=/home/data/github-mirror/dumps
+
+# Base URL for HTTP dir containing torrents and data 
+WEBSEED=http://ikaria.dmst.aueb.gr/ghtorrent/
+
+# Time to start dumping from
+if [ -r lastrun ]
+then
+	timeStart=`cat lastrun`
+else
+	timeStart=0
+fi
+
+# Time to end dumping
+if [ "$1" = "" ]
+then
+	timeEnd=`date +%s`
+else
+	timeEnd=`date -d "$1" +%s` || exit 1
+fi
+
+# Name used for the files
+dateName=`date -d @$timeEnd -u +'%Y-%m-%d'`
+
+# _id example:
+# 4f208c3e08d69a1835000077
+# 000102030405060708091011
+# |      ||    ||  ||    |
+# time    mach  pid count
+
+endId=`printf '%08x0000000000000000' $timeEnd`
+startId=`printf '%08x0000000000000000' $timeStart`
+
+echo "Dumping database from `date -d @$timeStart` to `date -d @$timeEnd`"
+
+collections=`echo "show collections"|mongo --quiet github|egrep -v "system|bye"`
+	
+rm -rf dump
+for col in $collections; do
+
+	echo "Dumping $col"
+	mongodump --db github --collection $col -q '{"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }' || exit 1
+done
+
+# Report the metadata for the given database
+meta()
+{
+	echo -n "Number of $1: "
+	mongo --quiet --eval 'db.'$1'.find({"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }).count() + 0' github
+	echo -n "Uncompressed size of $1: "
+	wc -c dump/github/$1.bson | awk '{printf "%d bytes ", $1}'
+	du -h dump/github/$1.bson | awk '{print " (" $1 ")" }'
+}
+
+for col in $collections; do
+(
+	echo "Start date: `date -u -d @$timeStart +'%Y-%m-%dT%H:%M:%SZ'`"
+	echo "End date: `date -u -d @$timeEnd +'%Y-%m-%dT%H:%M:%SZ'`"
+	meta $col 
+) 
+done |
+tee README.$dateName.txt >dump/github/README.txt || exit 1
+
+# Do the same per collection
+for col in $collections; do
+	echo "Archiving $col.bson"
+	if [ ! -s dump/github/$col.bson ]; then
+		echo "Collection empty, skipping"
+		continue
+	fi
+
+	if ! tar zcf $OUTDIR/$col-dump.$dateName.tar.gz dump/github/$col.bson
+	then
+		rm -f $OUTDIR/$col-dump.$dateName.tar.gz
+		exit 1
+	fi
+
+	mktorrent -a udp://tracker.openbittorrent.com:80 -a udp://tracker.publicbt.com:80/announce -a http://tracker.bittorrent.am/announce -w $WEBSEED/$col-dump.$dateName.tar.gz -o $OUTDIR/$col-dump.$dateName.torrent $OUTDIR/$col-dump.$dateName.tar.gz
+done
+
+# Update last run info
+echo $timeEnd >lastrun || exit 1
+
+# Clean up
+rm -rf dump 
+

data/bin/ght-rm-dupl

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+#
+# Knows how to remove duplicate entries from various collections.
+#
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'rubygems'
+require 'mongo'
+require 'ghtorrent-old'
+
+GH = Mirror.new
+GH.init("config.yaml")
+
+# Unique keys per known collection
+per_col = {
+    :commits => {
+        :payload => "commit.id",
+        :col => GH.commits_col,
+    },
+    :events => {
+        :payload => "id",
+        :col => GH.events_col,
+    }
+}
+
+# Print MongoDB remove statements that
+# remove all but one entries for each commit.
+def remove_duplicates(data, col)
+  removed = 0
+  data.select { |k, v| v.size > 1 }.each do |k, v|
+    v.slice(0..(v.size - 2)).map do |x|
+      removed += 1 if delete_by_id col, x
+    end
+  end
+  removed
+end
+
+def delete_by_id(col, id)
+  begin
+    col.remove({'_id' => id})
+    true
+  rescue Mongo::OperationFailure
+    puts "Cannot remove record with id #{id} from #{col.name}"
+    false
+  end
+end
+
+which = case ARGV[0]
+          when "commits" then :commits
+          when "events" then :events
+          else puts "Not a known collection name: #{ARGV[0]}\n"
+        end
+
+from = case ARGV[1]
+         when nil then {}
+         else
+           t = Time.at(ARGV[1].to_i)
+           STDERR.puts "Searching for duplicates after #{t}"
+           {'_id' => {'$gte' => BSON::ObjectId.from_time(t)}}
+       end
+
+# Various counters to report stats
+processed = total_processed = removed = 0
+
+data = Hash.new
+
+# The following code needs to save intermediate results to cope
+# with large datasets
+per_col[which][:col].find(from, :fields => per_col[which][:payload]).each do |r|
+  _id = r["_id"]
+  commit = GH.read_value(r, per_col[which][:payload])
+
+  # If entries cannot be parsed, remove them
+  if commit.empty?
+    puts "Deleting unknown entry #{_id}"
+    removed += 1 if delete_by_id per_col[which][:col], _id
+  else
+    data[commit] = [] if data[commit].nil?
+    data[commit] << _id
+  end
+
+  processed += 1
+  total_processed += 1
+
+  print "\rProcessed #{processed} records"
+
+  # Calculate duplicates, save intermediate result
+  if processed > 500000
+    puts "\nLoaded #{data.size} values, cleaning"
+    removed += remove_duplicates data, per_col[which][:col]
+    data = Hash.new
+    processed = 0
+  end
+end
+
+removed += remove_duplicates data, per_col[which][:col]
+
+puts "Processed #{total_processed}, deleted #{removed} duplicates"

data/bin/ght-torrent-index

@@ -0,0 +1,180 @@
+#!/usr/bin/env ruby
+#
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'rubygems'
+require 'erb'
+require 'set'
+require 'date'
+require 'ghtorrent'
+
+class Page
+  attr_reader :collections
+  attr_reader :dumps
+
+  def initialize(last_update)
+    @last_update = last_update
+    @dumps = Set.new
+    @collections = Set.new
+  end
+
+  def add_dump(dump)
+    @dumps << dump
+  end
+
+  def add_collection(col)
+    @collections << col
+  end
+
+  # Expose private binding() method.
+  def get_binding
+    binding()
+  end
+
+end
+
+class Dump
+  attr_reader :torrents
+  attr_reader :date
+
+  def initialize(torrents, date)
+    @torrents = torrents
+    @date = date
+  end
+end
+
+class Torrent
+  attr_reader :url
+  attr_reader :name
+  attr_reader :size
+  attr_reader :date
+
+  def initialize(url, name, size, date)
+    @url = url
+    @name = name
+    @size = size
+    @date = date
+  end
+end
+
+class Indexer < GHTorrent::Command
+
+  def prepare_options(options)
+    options.banner <<-BANNER
+Create an HTML table from a list of torrent and data files. The expected
+naming is the following:
+
+  collname-dump-2012-03-27.torrent
+  collname-dump-2012-03-27.tar.gz
+
+#{command_name} [options]
+
+#{command_name} options:
+        BANNER
+
+    options.opt :prefix, 'URL prefix to use for links',
+                :short => 'p', :default => "", :type => :string
+  end
+
+  def validate_options
+
+  end
+
+  def go
+    url_prefix=options[:prefix]
+
+    # Load the template
+    gem_root = Gem.loaded_specs['ghtorrent']
+
+    file = if gem_root.nil?
+             # Gem not installed yet, try current dir
+             File.open("index.erb").read
+           else
+             File.open(File.join(gem_root, "index.erb")).read
+           end
+
+    rhtml = ERB.new(file)
+
+    # Open the dir to read entries from
+    dir = ARGV.shift
+
+    if dir.nil?
+      dir = "."
+    end
+
+    torrents = Dir.entries("#{dir}").map do |f|
+
+      #File name format expected: collname-dump-2012-03-27.torrent
+      #                           collname-dump-2012-03-27.tar.gz
+
+      # Go through all torrent files and extract name of
+      # dumped collection and dump date
+      matches = /([a-z0-9]+)-[a-z]+\.(.*)\.torrent/.match(f)
+      next if matches.nil?
+
+      # Calculate original file size
+      dump = f.gsub(/.torrent/, ".tar.gz")
+      size = File.stat(File.join(dir, dump)).size / 1024 / 1024
+
+      # Expects a format of yyyy-mm-dd
+      date = Date.parse(matches[2])
+
+      if size > 0
+        Torrent.new(url_prefix + "/" + f, matches[1], size, date)
+      end
+    end.select { |x| !x.nil? }
+
+    all_dates = torrents.inject(Set.new) { |acc, t| acc << t.date }
+
+    all_dumps = all_dates.map { |d|
+      date_torrents = torrents.select { |t| t.date == d }
+      name_torrents = date_torrents.inject(Hash.new) { |acc, a|
+        acc.store(a.name, a);
+        acc
+      }
+      Dump.new(name_torrents, d)
+    }
+
+    max_date = all_dates.max { |a, b| a <=> b }
+
+    ghtorrent = Page.new(max_date)
+    all_dumps.each { |x|
+      ghtorrent.add_dump x
+      x.torrents.values.each { |t|
+        ghtorrent.add_collection t.name
+      }
+    }
+
+    puts rhtml.result(ghtorrent.get_binding).gsub(/^\s+/, "").gsub(/\s+$/, $/).gsub(/<table>/, "\n<table>")
+  end
+end
+
+Indexer.run
+
+# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :