ghtorrent 0.6 → 0.7.1

data/bin/ght-periodic-dump

@@ -1,130 +0,0 @@
-#!/bin/sh
-#
-# Create the periodic database dump files
-#
-
-# Directory to place compressed files and torrents
-OUTDIR=/home/data/github-mirror/dumps
-
-# Base URL for HTTP dir containing torrents and data
-WEBSEED=http://ikaria.dmst.aueb.gr/ghtorrent/
-
-usage()
-{
-	echo "Usage: $0 [-f 'yyyy-mm-dd hh:mm'] [-t 'yyyy-mm-dd hh:mm']"
-	echo "          [-c collection_to_dump]"
-	echo "Dump the database. -f earliest record timestamp"
-	echo "                   -t latest record timestamp"
-	echo "                   -c collection to dump (default: all)"
-}
-
-if [ -z $1 ]
-then
-	usage
-	exit 1
-fi
-
-while getopts "f:t:c:" o
-do
-	case $o in
-	f) 	timeStart=`date -d "$OPTARG" +%s` ;;
-	t) 	timeEnd=`date -d "$OPTARG" +%s` ;;
-	c) 	collection=$OPTARG ;;
-	\?)     echo "Invalid option: -$OPTARG" >&2
-		usage
-	        exit 1
-		;;
-	esac
-done
-
-
-# Time to start dumping from
-if [ -z $timeStart ]
-then
-	if [ -r lastrun ]
-	then
-		timeStart=`cat lastrun`
-	else
-		timeStart=0
-	fi
-fi
-
-# Time to end dumping
-if [ -z $timeEnd ]
-then
-	timeEnd=`date +%s`
-fi
-
-# Name used for the files
-dateName=`date -d @$timeEnd -u +'%Y-%m-%d'`
-
-# _id example:
-# 4f208c3e08d69a1835000077
-# 000102030405060708091011
-# |      ||    ||  ||    |
-# time    mach  pid count
-
-endId=`printf '%08x0000000000000000' $timeEnd`
-startId=`printf '%08x0000000000000000' $timeStart`
-
-
-if [ -z $collection ]
-then
-	collections=`echo "show collections"|mongo --quiet github|egrep -v "system|bye"`
-else
-	collections=$collection
-fi
-
-echo "Dumping database from `date -d @$timeStart` to `date -d @$timeEnd`"
-
-rm -rf dump
-mkdir -p dump/github
-
-for col in $collections; do
-
-	echo "Dumping $col"
-	mongodump --db github --collection $col -q '{"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }' || exit 1
-done
-
-# Report the metadata for the given database
-meta()
-{
-	echo -n "Number of $1: "
-	mongo --quiet --eval 'db.'$1'.find({"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }).count() + 0' github
-	echo -n "Uncompressed size of $1: "
-	wc -c dump/github/$1.bson | awk '{printf "%d bytes ", $1}'
-	du -h dump/github/$1.bson | awk '{print " (" $1 ")" }'
-}
-
-for col in $collections; do
-(
-	echo "Start date: `date -u -d @$timeStart +'%Y-%m-%dT%H:%M:%SZ'`"
-	echo "End date: `date -u -d @$timeEnd +'%Y-%m-%dT%H:%M:%SZ'`"
-	meta $col
-)
-done |
-tee README.$dateName.txt >dump/github/README.txt || exit 1
-
-# Do the same per collection
-for col in $collections; do
-	echo "Archiving $col.bson"
-	if [ ! -s dump/github/$col.bson ]; then
-		echo "Collection empty, skipping"
-		continue
-	fi
-
-	if ! tar zcf $OUTDIR/$col-dump.$dateName.tar.gz dump/github/$col.bson
-	then
-		rm -f $OUTDIR/$col-dump.$dateName.tar.gz
-		exit 1
-	fi
-
-	mktorrent -a udp://tracker.openbittorrent.com:80 -a udp://tracker.publicbt.com:80/announce -a http://tracker.bittorrent.am/announce -w $WEBSEED/$col-dump.$dateName.tar.gz -o $OUTDIR/$col-dump.$dateName.torrent $OUTDIR/$col-dump.$dateName.tar.gz
-done
-
-# Update last run info
-echo $timeEnd >lastrun || exit 1
-
-# Clean up
-rm -rf dump
-

data/bin/ght-torrent-index

@@ -1,150 +0,0 @@
-require 'rubygems'
-require 'erb'
-require 'set'
-require 'date'
-require 'ghtorrent'
-
-class Page
-  attr_reader :collections
-  attr_reader :dumps
-
-  def initialize(last_update)
-    @last_update = last_update
-    @dumps = Set.new
-    @collections = Set.new
-  end
-
-  def add_dump(dump)
-    @dumps << dump
-  end
-
-  def add_collection(col)
-    @collections << col
-  end
-
-  # Expose private binding() method.
-  def get_binding
-    binding()
-  end
-
-end
-
-class Dump
-  attr_reader :torrents
-  attr_reader :date
-
-  def initialize(torrents, date)
-    @torrents = torrents
-    @date = date
-  end
-end
-
-class Torrent
-  attr_reader :url
-  attr_reader :name
-  attr_reader :size
-  attr_reader :date
-
-  def initialize(url, name, size, date)
-    @url = url
-    @name = name
-    @size = size
-    @date = date
-  end
-end
-
-class Indexer < GHTorrent::Command
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Create an HTML table from a list of torrent and data files. The expected
-naming is the following:
-
-  collname-dump-2012-03-27.torrent
-  collname-dump-2012-03-27.tar.gz
-
-#{command_name} [options]
-
-#{command_name} options:
-        BANNER
-
-    options.opt :prefix, 'URL prefix to use for links',
-                :short => 'p', :default => "", :type => :string
-  end
-
-  def validate_options
-
-  end
-
-  def go
-    url_prefix=options[:prefix]
-
-    # Load the template
-    gem_root = Gem.loaded_specs['ghtorrent']
-
-    file = if gem_root.nil?
-             # Gem not installed yet, try current dir
-             File.open("index.erb").read
-           else
-             File.open(File.join(gem_root, "index.erb")).read
-           end
-
-    rhtml = ERB.new(file)
-
-    # Open the dir to read entries from
-    dir = ARGV.shift
-
-    if dir.nil?
-      dir = "."
-    end
-
-    torrents = Dir.entries("#{dir}").map do |f|
-
-      #File name format expected: collname-dump-2012-03-27.torrent
-      #                           collname-dump-2012-03-27.tar.gz
-
-      # Go through all torrent files and extract name of
-      # dumped collection and dump date
-      matches = /([a-z0-9]+)-[a-z]+\.(.*)\.torrent/.match(f)
-      next if matches.nil?
-
-      # Calculate original file size
-      dump = f.gsub(/.torrent/, ".tar.gz")
-      size = File.stat(File.join(dir, dump)).size / 1024 / 1024
-
-      # Expects a format of yyyy-mm-dd
-      date = Date.parse(matches[2])
-
-      if size > 0
-        Torrent.new(url_prefix + "/" + f, matches[1], size, date)
-      end
-    end.select { |x| !x.nil? }
-
-    all_dates = torrents.inject(Set.new) { |acc, t| acc << t.date }
-
-    all_dumps = all_dates.map { |d|
-      date_torrents = torrents.select { |t| t.date == d }
-      name_torrents = date_torrents.inject(Hash.new) { |acc, a|
-        acc.store(a.name, a);
-        acc
-      }
-      Dump.new(name_torrents, d)
-    }
-
-    max_date = all_dates.max { |a, b| a <=> b }
-
-    ghtorrent = Page.new(max_date)
-    all_dumps.each { |x|
-      ghtorrent.add_dump x
-      x.torrents.values.each { |t|
-        ghtorrent.add_collection t.name
-      }
-    }
-
-    puts rhtml.result(ghtorrent.get_binding).gsub(/^\s+/, "").gsub(/\s+$/, $/).gsub(/<table>/, "\n<table>")
-  end
-end
-
-Indexer.run
-
-# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/test/callstack_test.rb

@@ -1,67 +0,0 @@
-require "test/unit"
-require 'ghtorrent'
-
-class CallStackTest < Test::Unit::TestCase
-
-  def setup
-  end
-
-  def teardown
-  end
-
-  def test_constructor
-    a = CallStack.new('users', 0)
-    b = CallStack.new('users', 0)
-    assert_equal a,b
-  end
-
-  def test_push
-    stack = CallStack.new('users1', 0)
-    assert_not_nil stack
-
-    stack.push("foo bar")
-    stack.push("2")
-    stack.push("1234421")
-    stack.empty
-  end
-
-  def test_pop
-    stack = CallStack.new('users2', 0)
-    assert_not_nil stack
-
-    stack.push("foo bar")
-    stack.push("2")
-    stack.push("1234421")
-
-    assert stack.pop == "1234421"
-    stack.empty
-  end
-
-  def test_push_pop_push
-    stack = CallStack.new('users3', 0)
-    assert_not_nil stack
-
-    stack.push("foo bar")
-    stack.push("2")
-
-    stack.pop
-
-    stack.push("1234421")
-
-    stack.empty
-  end
-
-  def test_stress
-    stack = CallStack.new('users4', 0)
-
-    1000.times do
-      txt = (0..rand(20)).map{65.+(rand(25)).chr}.join
-      stack.push txt
-    end
-
-    999.times do
-      stack.pop
-    end
-    stack.pop
-  end
-end