ghtorrent 0.2

data/lib/ghtorrent/logging.rb

@@ -0,0 +1,69 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'logger'
+
+module GHTorrent
+  module Logging
+
+    DEBUG_LEVEL = defined?(Logger) ? Logger::DEBUG : 0
+
+    def warn(msg)
+      log(:warn, msg)
+    end
+
+    def info(msg)
+      log(:info, msg)
+    end
+
+    def debug(msg)
+      log(:debug, msg)
+    end
+
+    private
+
+    # Log a message at the given level.
+    def log(level, msg)
+      return unless @logger
+      case level
+        when :fatal then
+          @logger.fatal msg
+        when :error then
+          @logger.error msg
+        when :warn then
+          @logger.warn msg
+        when :info then
+          @logger.info msg
+        when :debug then
+          @logger.debug msg
+        else
+          @logger.debug msg
+      end
+    end
+  end
+end

data/lib/ghtorrent/migrations/001_init_schema.rb

@@ -0,0 +1,60 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+    puts("Creating table users")
+    create_table :users do
+      primary_key :id
+      String :login, :unique => true, :null => false
+      String :name
+      String :company, :null => true
+      String :location, :null => true
+      String :email, :null => true, :unique => true
+      TrueClass :hireable, :null => true
+      String :bio, :null => true
+      Time :created_at, :null => false
+    end
+
+    puts("Creating table projects")
+    create_table :projects do
+      primary_key :id
+      String :url
+      foreign_key :owner_id, :users
+      String :name, :null => false
+      String :description
+      String :language
+      Time :created_at, :null => false
+    end
+
+    puts("Creating table commits")
+    create_table :commits do
+      primary_key :id
+      String :sha, :size => 40, :unique => true
+      foreign_key :author_id, :users
+      foreign_key :committer_id, :users
+      Time :created_at, :null => false
+    end
+
+    puts("Creating table commit_parents")
+    create_table :commit_parents do
+      foreign_key :commit_id, :commits, :null => false
+      foreign_key :parent_id, :commits, :null => false
+      primary_key [:commit_id, :parent_id]
+    end
+
+    puts("Creating table followers")
+    create_table :followers do
+      foreign_key :user_id, :users, :null => false
+      foreign_key :follower_id, :users, :null => false
+      primary_key [:user_id, :follower_id]
+    end
+  end
+
+  down do
+    drop_table :users
+    drop_table :projects
+    drop_table :commits
+    drop_table :commit_parents
+    drop_table :followers
+  end
+end

data/lib/ghtorrent/migrations/002_add_followers_created_at.rb

@@ -0,0 +1,15 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+    alter_table :followers do
+      add_column :created_at, :Time, :null => false, :default => Time.now
+    end
+  end
+
+  down do
+    alter_table :followers do
+      drop_column :created_at
+    end
+  end
+end

data/lib/ghtorrent/migrations/003_add_external_ref_ids.rb

@@ -0,0 +1,40 @@
+require 'sequel'
+
+Sequel.migration do
+  up do
+    alter_table :users do
+      add_column :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    end
+
+    alter_table :projects do
+      add_column :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    end
+
+    alter_table :commits do
+      add_column :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    end
+
+    alter_table :followers do
+      add_column :ext_ref_id, String, :null => false, :size => 24, :default => "0"
+    end
+  end
+
+  down do
+    alter_table :users do
+      drop_column :ext_ref_id
+    end
+
+    alter_table :projects do
+      drop_column :ext_ref_id
+    end
+
+    alter_table :commits do
+      drop_column :ext_ref_id
+    end
+
+    alter_table :followers do
+      drop_column :ext_ref_id
+    end
+  end
+end
+

data/lib/ghtorrent/persister.rb

@@ -0,0 +1,48 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+module GHTorrent
+
+  #
+  module Persister
+
+    ADAPTERS = {
+      :mongo => GHTorrent::MongoPersister,
+      :noop => GHTorrent::NoopPersister
+    }
+
+    # Factory method for retrieving persistence connections.
+    # The +settings+ argument is a fully parsed YAML document
+    # passed on to adapters. The available +adapter+ are :mongo and :noop
+    def connect(adapter, settings)
+      driver = ADAPTERS[adapter]
+      driver.new(settings)
+    end
+
+  end
+end

data/lib/ghtorrent/retriever.rb

@@ -0,0 +1,148 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+module GHTorrent
+  module Retriever
+
+    include GHTorrent::APIClient
+    include GHTorrent::Settings
+
+    def initialize(settings)
+      super(settings)
+      @settings = settings
+      @uniq = config(:uniq_id)
+    end
+
+    def retrieve_user_byusername(user)
+      stored_user = @persister.find(:users, {'login' => user})
+      if stored_user.empty?
+        url = ghurl "users/#{user}"
+        u = api_request(url)
+
+        if u.nil?
+          throw GHTorrentException.new("Cannot find user #{user}")
+        end
+
+        unq = @persister.store(:users, u)
+        u[@uniq] = unq
+        info "Retriever: New user #{user}"
+        u
+      else
+        debug "Retriever: Already got user #{user}"
+        stored_user.first
+      end
+    end
+
+    # Try Github API v2 user search by email. This is optional info, so
+    # it may not return any data.
+    # http://develop.github.com/p/users.html
+    def retrieve_user_byemail(email, name)
+      url = ghurl_v2("user/email/#{email}")
+      api_request(url)
+    end
+
+    def retrieve_new_user_followers(user)
+      stored_followers = @persister.find(:followers, {'follows' => user})
+
+      followers = paged_api_request(ghurl "users/#{user}/followers")
+      followers.each do |x|
+        x['follows'] = user
+
+        exists = !stored_followers.find { |f|
+          f['follows'] == user && f['login'] == x['login']
+        }.nil?
+
+        if not exists
+          @persister.store(:followers, x)
+          info "Retriever: Added follower #{user} -> #{x['login']}"
+        else
+          debug "Retriever: Follower #{user} -> #{x['login']} exists"
+        end
+      end
+
+      @persister.find(:followers, {'follows' => user})
+    end
+
+    def retrieve_commit(repo, sha, user)
+      commit = @persister.find(:commits, {'sha' => "#{sha}"})
+
+      if commit.empty?
+        url = ghurl "repos/#{user}/#{repo}/commits/#{sha}"
+        c = api_request(url)
+
+        if c.nil?
+          throw GHTorrentException.new("Cannot find commit #{user}/#{repo}/#{sha}")
+        end
+
+        unq = @persister.store(:commits, c)
+        info "Retriever: New commit #{repo} -> #{sha}"
+        c[@uniq] = unq
+        c
+      else
+        debug "Retriever: Already got commit #{repo} -> #{sha}"
+        commit.first
+      end
+    end
+
+    def retrieve_repo(user, repo)
+      stored_repo = @persister.find(:repos, {'owner.login' => user,
+                                             'name' => repo })
+      if stored_repo.empty?
+        url = ghurl "repos/#{user}/#{repo}"
+        r = api_request(url)
+
+        if r.nil?
+          throw GHTorrentException.new("Cannot find repo #{user}/#{repo}")
+        end
+
+        unq = @persister.store(:repos, r)
+        info "Retriever: New repo #{user} -> #{repo}"
+        r[@uniq] = unq
+        r
+      else
+        debug "Retriever: Already got repo #{user} -> #{repo}"
+        stored_repo.first
+      end
+    end
+
+    # Get current Github events
+    def get_events
+      api_request "https://api.github.com/events"
+    end
+
+    private
+
+    def ghurl(path)
+      config(:mirror_urlbase) + path
+    end
+
+    def ghurl_v2(path)
+      config(:mirror_urlbase_v2) + path
+    end
+  end
+end

data/lib/ghtorrent/settings.rb

@@ -0,0 +1,63 @@
+# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'yaml'
+
+module GHTorrent
+  module Settings
+
+    include GHTorrent::Utils
+
+    CONFIGKEYS = {
+        :amqp_host => "amqp.host",
+        :amqp_port => "amqp.port",
+        :amqp_username => "amqp.username",
+        :amqp_password => "amqp.password",
+        :amqp_exchange => "amqp.exchange",
+
+        :sql_url => "sql.url",
+
+        :mirror_urlbase => "mirror.urlbase",
+        :mirror_urlbase_v2 => "mirror.urlbase_v2",
+        :mirror_reqrate => "mirror.reqrate",
+        :mirror_pollevery => "mirror.pollevery",
+        :mirror_persister => "mirror.persister",
+
+        :uniq_id => "uniq_id"
+    }
+
+    def config(key)
+      read_value(settings, CONFIGKEYS[key])
+    end
+
+    def merge(more_keys)
+      more_keys.each {|k,v| CONFIGKEYS[k] = v}
+    end
+
+  end
+end