ghtorrent 0.2

data/lib/ghtorrent.rb

@@ -0,0 +1,22 @@
+#require 'ghtorrent-old/ghtorrent-old'
+
+module GHTorrent
+  VERSION = 0.2
+end
+
+require 'ghtorrent/command'
+
+require 'ghtorrent/utils'
+require 'ghtorrent/logging'
+require 'ghtorrent/settings'
+require 'ghtorrent/api_client'
+require 'ghtorrent/call_stack'
+
+require 'ghtorrent/adapters/base_adapter'
+require 'ghtorrent/adapters/mongo_persister'
+require 'ghtorrent/adapters/noop_persister'
+
+require 'ghtorrent/persister'
+require 'ghtorrent/retriever'
+
+require 'ghtorrent/ghtorrent'

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -0,0 +1,91 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+module GHTorrent
+
+  class BaseAdapter
+
+    ENTITIES = [:users, :commits, :followers, :repos, :events]
+
+
+    # Stores +data+ into +entity+. Returns a unique key for the stored entry.
+    def store(entity, data = {})
+      unless ENTITIES.include?(entity)
+        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+      end
+    end
+
+    # Retrieves rows from +entity+ matching the provided +query+.
+    # The +query+
+    # is performed on the Github API JSON results. For example, given the
+    # following JSON object format:
+    #
+    #   {
+    #      commit: {
+    #        sha: "23fa34aa442456"
+    #      }
+    #      author: {
+    #        name: {
+    #          real_name: "foo"
+    #          given_name: "bar"
+    #        }
+    #      }
+    #      created_at: "1980-12-30T22:25:25"
+    #   }
+    #
+    # to query for matching +sha+, pass to +query+
+    #
+    #   {'commit.sha' => 'a_value'}
+    #
+    # to query for real_name's matching an argument, pass to +query+
+    #
+    #   {'author.name.real_name' => 'a_value'}
+    #
+    # to query for both a specific sha and a specific creation time
+    #
+    #   {'commit.sha' => 'a_value', 'created_at' => 'other_value'}
+    #
+    # The persister adapter must translate the query to the underlying data
+    # storage engine query capabilities.
+    #
+    # The results are returned as an array of hierarchical maps, one for each
+    # matching JSON object.
+    def find(entity, query = {})
+      unless ENTITIES.include?(entity)
+        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+      end
+    end
+
+    # Find the record identified by +id+ in +entity+
+    def find_by_ext_ref_id(entity, id)
+      unless ENTITIES.include?(entity)
+        throw GHTorrentException.new("Perister: Entity #{entity} not known")
+      end
+    end
+  end
+end

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -0,0 +1,126 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'mongo'
+
+module GHTorrent
+
+  # A persistence adapter that saves data into a configurable MongoDB database.
+  class MongoPersister < GHTorrent::BaseAdapter
+
+    include GHTorrent::Settings
+    include GHTorrent::Logging
+
+    # Supported configuration options.
+    LOCALCONFIG = {
+        :mongo_host => "mongo.host",
+        :mongo_port => "mongo.port",
+        :mongo_db => "mongo.db",
+        :mongo_username => "mongo.username",
+        :mongo_passwd => "mongo.password"
+    }
+
+    attr_reader :settings
+
+    # Creates a new instance of the MongoDB persistence adapter.
+    # Expects a parsed YAML settings document as input.
+    def initialize(set)
+      merge LOCALCONFIG
+
+      @settings = set
+      @uniq = config(:uniq_id)
+      @mongo = Mongo::Connection.new(config(:mongo_host),
+                                     config(:mongo_port))\
+                                .db(config(:mongo_db))
+      @enttodb = {
+          :users => get_collection("users"),
+          :commits => get_collection("commits"),
+          :repos => get_collection("repos"),
+          :followers => get_collection("followers"),
+          :events => get_collection("events")
+      }
+    end
+
+
+    def store(entity, data = {})
+      super
+      col = @enttodb[entity]
+
+      if col.nil?
+        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
+      end
+
+      col.insert(data).to_s
+    end
+
+    def find(entity, query = {})
+      super
+
+      col = @enttodb[entity]
+
+      if col.nil?
+        raise GHTorrentException.new("Mongo: Entity #{entity} not supported")
+      end
+
+      result = col.find(query)
+      result.to_a.map { |r|
+        r[@uniq] = r['_id'].to_s;
+        r.to_h
+      }
+    end
+
+    # Find the record identified by +id+ in +entity+
+    def find_by_ext_ref_id(entity, id)
+      super
+      raise NotImplementedError
+    end
+
+    private
+
+    def get_collection(col)
+      @mongo.collection(col.to_s)
+    end
+
+  end
+end
+
+class BSON::OrderedHash
+
+  # Convert a BSON result to a +Hash+
+  def to_h
+    inject({}) do |acc, element|
+      k, v = element;
+      acc[k] = if v.class == BSON::OrderedHash then
+                 v.to_h
+               else
+                 v
+               end;
+      acc
+    end
+  end
+end

data/lib/ghtorrent/adapters/noop_persister.rb

@@ -0,0 +1,58 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+module GHTorrent
+
+  # Persister adapter that does not store any data.
+  class NoopPersister < BaseAdapter
+
+    def init(settings)
+    end
+
+    def store(entity, data = {})
+      super
+      #Nothing to see here
+      0
+    end
+
+    def find(entity, query = {})
+      super
+      #Nothing to see here
+      []
+    end
+
+    def find_by_ext_ref_id(entity, id)
+      super
+      nil
+    end
+
+    def get_id
+      0
+    end
+  end
+end

data/lib/ghtorrent/api_client.rb

@@ -0,0 +1,106 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'net/http'
+require 'set'
+require 'open-uri'
+require 'json'
+
+module GHTorrent
+  module APIClient
+    include GHTorrent::Logging
+    include GHTorrent::Settings
+
+    def initialize(settings)
+      @num_api_calls = 0
+      @ts = Time.now().tv_sec()
+    end
+    
+    def paged_api_request(url, pages = -1)
+
+      pg = if pages == -1 then
+             1000000
+           else
+             pages
+           end
+      result = Array.new
+
+      (1..pg).each { |x|
+        data = api_request("#{url}?page=#{x}")
+        result += data
+        break if data.empty?
+      }
+      result
+    end
+
+    def api_request(url)
+      result = api_request_raw(url)
+      if result.nil?
+        nil
+      else
+        JSON.parse(result)
+      end
+    end
+
+    def api_request_raw(url)
+      #Rate limiting to avoid error requests
+      if Time.now().tv_sec() - @ts < 60 then
+        if @num_api_calls >= @settings['mirror']['reqrate'].to_i
+          sleep = 60 - (Time.now().tv_sec() - @ts)
+          debug "APIClient: Sleeping for #{sleep}"
+          sleep (sleep)
+          @num_api_calls = 0
+          @ts = Time.now().tv_sec()
+        end
+      else
+        debug "APIClient: Tick, num_calls = #{@num_api_calls}, zeroing"
+        @num_api_calls = 0
+        @ts = Time.now().tv_sec()
+      end
+
+      @num_api_calls += 1
+      debug "APIClient: Request: #{url} (num_calls = #{@num_api_calls})"
+      begin
+        open(url).read
+      rescue OpenURI::HTTPError => e
+        case e.io.status[0].to_i
+          # The following indicate valid Github return codes
+          when 400, # Bad request
+              401, # Unauthorized
+              403, # Forbidden
+              404, # Not found
+              422 : # Unprocessable entity
+            STDERR.puts "#{url}: #{e.io.status[1]}"
+            return nil
+          else # Server error or HTTP conditions that Github does not report
+            raise e
+        end
+      end
+    end
+  end
+end

data/lib/ghtorrent/call_stack.rb

@@ -0,0 +1,119 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+module GHTorrent
+  class CallStack
+
+    @@callstacks = Hash.new
+
+    attr_reader :name
+
+    def self.new(*args)
+      name = args[0]
+      if @@callstacks.has_key? name
+        @@callstacks[name]
+      else
+        o = allocate
+        if o.__send__(:initialize, *args)
+          @@callstacks[name] = o
+          o
+        else
+          nil
+        end
+      end
+    end
+
+    def initialize(name, sync_every = 5)
+
+      @stack = Array.new
+      @name = name
+      @sync = sync_every
+
+      if File.exists?(name)
+        @file = File.new(name, "r")
+        puts "File #{name} exists, importing stack..."
+        read = @file.readlines.reverse.reduce(0) { |acc, x|
+          @stack.push x
+          acc
+        }
+        puts "\n#{read} entries read"
+        @file.close
+      end
+
+      flusher = Thread.new {
+        while true
+          begin
+            if not @stack.empty?
+              @file = File.new(name, "w+")
+              @stack.each { |l| @file.write("#{l} \n") }
+              @file.fsync
+              @file.close
+            end
+            sleep(@sync)
+          rescue
+            puts "flusher thread failed for #{name}"
+          end
+        end
+      }
+
+      ObjectSpace.define_finalizer(self, proc {
+        puts "Finalizer: Cleaning up #{@name}"
+        @@callstacks.delete[@name]
+        flusher.stop
+        cleanup
+      })
+
+      at_exit { cleanup }
+    end
+
+    def push(item)
+      @stack.push(item)
+    end
+
+    def pop()
+      @stack.pop
+    end
+
+    def empty
+      @stack.delete_if { |x| true }
+    end
+
+    private
+
+    def cleanup
+      if @stack.empty?
+        if File.exists? @name
+          puts "removing stack #{@name}"
+          File.delete(@name)
+        end
+      else
+        puts "stack #{@name} contains #{@stack.size} items"
+      end
+    end
+  end
+end