ghtorrent 0.4 → 0.5

data/lib/ghtorrent/api_client.rb

@@ -1,28 +1,41 @@
-require 'net/http'
-require 'set'
 require 'open-uri'
+require 'net/http'
+require 'digest/sha1'
+require 'fileutils'
 require 'json'
 
+require 'ghtorrent/logging'
+require 'ghtorrent/settings'
+require 'ghtorrent/time'
+require 'ghtorrent/cache'
+
 module GHTorrent
   module APIClient
     include GHTorrent::Logging
     include GHTorrent::Settings
+    include GHTorrent::Cache
 
-    def initialize(settings)
-      @num_api_calls = 0
-      @ts = Time.now().tv_sec()
+    # This is to fix an annoying bug in JRuby's SSL not being able to
+    # verify a valid certificate.
+    if defined? JRUBY_VERSION
+      OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
     end
 
     # A paged request. Used when the result can expand to more than one
     # result pages.
-    def paged_api_request(url, pages = -1)
+    def paged_api_request(url, pages = -1, cache = true, last = nil)
 
-      data = api_request_raw(url)
+      data = if URI.parse(url).query.nil? # Top level request, no params
+               api_request_raw(url, false)
+             else
+               api_request_raw(url, use_cache?(cache, method = :paged))
+             end
 
       return [] if data.nil?
 
       unless data.meta['link'].nil?
         links = parse_links(data.meta['link'])
+        last = links['last'] if last.nil?
 
         if pages > 0
           pages = pages - 1
@@ -34,7 +47,15 @@ module GHTorrent
         if links['next'].nil?
           parse_request_result(data)
         else
-          parse_request_result(data) | paged_api_request(links['next'], pages)
+          parse_request_result(data) |
+              if links['next'] == last
+                if last != links['last']
+                  warn "APIClient: Last header mismatch: method=#{last}, cache=#{links['last']}"
+                end
+                paged_api_request(links['next'], pages, false, last)
+              else
+                paged_api_request(links['next'], pages, cache, last)
+              end
         end
       else
         parse_request_result(data)
@@ -43,12 +64,43 @@ module GHTorrent
 
     # A normal request. Returns a hash or an array of hashes representing the
     # parsed JSON result.
-    def api_request(url)
-      parse_request_result api_request_raw(url)
+    def api_request(url, cache = false)
+      parse_request_result api_request_raw(url, use_cache?(cache))
     end
 
     private
 
+    # Determine whether to use cache or not, depending on the type of the
+    # request
+    def use_cache?(client_request, method = :non_paged)
+      @cache_mode ||= case config(:cache_mode)
+                        when "dev"
+                          :dev
+                        when "prod"
+                          :prod
+                        else
+                          raise GHTorrentException("")
+                      end
+      case @cache_mode
+        when :dev
+          unless client_request
+            return false
+          end
+          return true
+        when :prod
+          if client_request
+            return true
+          else
+            case method
+              when :non_paged
+                return false
+              when :paged
+                return true
+            end
+          end
+      end
+    end
+
     # Parse a Github link header
     def parse_links(links)
       links.split(/,/).reduce({}) do |acc, x|
@@ -64,6 +116,7 @@ module GHTorrent
         []
       else
         json = result.read
+
         if json.nil?
           []
         else
@@ -73,7 +126,10 @@ module GHTorrent
     end
 
     # Do the actual request and return the result object
-    def api_request_raw(url)
+    def api_request_raw(url, use_cache = false)
+      @num_api_calls ||= 0
+      @ts ||= Time.now().tv_sec()
+
       #Rate limiting to avoid error requests
       if Time.now().tv_sec() - @ts < 60 then
         if @num_api_calls >= @settings['mirror']['reqrate'].to_i
@@ -89,10 +145,29 @@ module GHTorrent
         @ts = Time.now().tv_sec()
       end
 
-      @num_api_calls += 1
-      debug "APIClient: Request: #{url} (num_calls = #{@num_api_calls})"
       begin
-        open(url)
+        start_time = Time.now
+        from_cache = false
+
+        contents =
+            if use_cache
+              if not (cached = cache_get(url)).nil?
+                from_cache = true
+                cached
+              else
+                tocache = Cachable.new(do_request(url))
+                @num_api_calls += 1
+                cache_put(url, tocache)
+                tocache
+              end
+            else
+              @num_api_calls += 1
+              do_request(url)
+            end
+
+        total = Time.now.to_ms - start_time.to_ms
+        debug "APIClient: Request: #{url} (#{@num_api_calls} calls,#{if from_cache then " from cache," end} Total: #{total} ms)"
+        contents
       rescue OpenURI::HTTPError => e
         case e.io.status[0].to_i
           # The following indicate valid Github return codes
@@ -100,7 +175,7 @@ module GHTorrent
               401, # Unauthorized
               403, # Forbidden
               404, # Not found
-              422 : # Unprocessable entity
+              422 then # Unprocessable entity
             STDERR.puts "#{url}: #{e.io.status[1]}"
             return nil
           else # Server error or HTTP conditions that Github does not report
@@ -109,5 +184,65 @@ module GHTorrent
         end
       end
     end
+
+    def do_request(url)
+      @attach_ip ||= config(:attach_ip)
+
+      if @attach_ip.nil? or @attach_ip.eql? "0.0.0.0"
+        open(url)
+      else
+        attach_to(@attach_ip) do
+          open(url)
+        end
+      end
+    end
+
+    # Attach to a specific IP address if the machine has multiple
+    def attach_to(ip)
+      TCPSocket.instance_eval do
+        (class << self; self; end).instance_eval do
+          alias_method :original_open, :open
+
+          define_method(:open) do |conn_address, conn_port|
+            original_open(conn_address, conn_port, ip)
+          end
+        end
+      end
+
+      result = begin
+        yield
+      rescue Exception => e
+        raise e
+      ensure
+        TCPSocket.instance_eval do
+          (class << self; self; end).instance_eval do
+            alias_method :open, :original_open
+            remove_method :original_open
+          end
+        end
+      end
+
+      result
+    end
+
+  end
+end
+
+class Cachable
+
+  include OpenURI::Meta
+
+  attr_reader :base_uri, :meta, :status
+
+  def initialize(response)
+    @data = response.read
+    @base_uri = response.base_uri
+    @meta = response.meta
+    @status = response.status
   end
-end
+
+  def read
+    @data
+  end
+
+end

data/lib/ghtorrent/bson_orderedhash.rb

@@ -0,0 +1,23 @@
+require 'json'
+
+class BSON::OrderedHash
+
+  # Convert a BSON result to a +Hash+
+  def to_h
+    inject({}) do |acc, element|
+      k, v = element;
+      acc[k] = if v.class == Array then
+                 v.map{|x| if x.class == BSON::OrderedHash then x.to_h else x end}
+               elsif v.class == BSON::OrderedHash then
+                 v.to_h
+               else
+                 v
+               end;
+      acc
+    end
+  end
+
+  def to_json
+    to_h.to_json
+  end
+end

data/lib/ghtorrent/cache.rb

@@ -0,0 +1,97 @@
+require 'digest/sha1'
+require 'fileutils'
+
+require 'ghtorrent/logging'
+require 'ghtorrent/settings'
+
+module GHTorrent
+  module Cache
+    include GHTorrent::Logging
+    include GHTorrent::Settings
+
+    # Root dir for cached objects.
+    def cache_dir
+      @cache_dir ||= config(:cache_dir)
+      @cache_dir
+    end
+
+    # The maximum time an item can be cached before being considered stale
+    def max_life
+      @max_life ||= config(:cache_stale_age)
+      @max_life
+    end
+
+    # Put an object to the cache
+    def cache_put(key, object)
+      file = cache_location(key)
+      FileUtils.mkdir_p(File.dirname (file))
+
+      begin
+        File.open(file, 'w') do |f|
+          f.flocked? do
+            YAML::dump object, f
+          end
+        end
+      rescue
+        warn "Could not cache object #{file} for key #{key}"
+      end
+    end
+
+    # Get the object indexed by +key+ from the cache. Returns nil if the
+    # key is not found or the object is too old.
+    def cache_get(key)
+      file = cache_location(key)
+
+      unless File.exist?(file)
+        return nil
+      end
+
+      unless (Time.now() - File.mtime(file)) < max_life
+        debug "Cached object for key #{key} too old"
+        return nil
+      end
+
+      begin
+        File.open(file, 'r') do |f|
+          f.flocked? do
+            YAML::load(f)
+          end
+        end
+      rescue
+        warn "Could not read object from cache location #{file}"
+        File.delete(file)
+      end
+    end
+
+    private
+
+    def cache_location(key)
+      hash = hashkey(key)
+      start = hash[0,2]
+      File.join(cache_dir, start, hash)
+    end
+
+    def hashkey(key)
+      Digest::SHA1.hexdigest key
+    end
+
+  end
+end
+
+class File
+  def flocked? &block
+    status = flock LOCK_EX
+    case status
+      when false
+        return true
+      when 0
+        begin
+          block ? block.call : false
+        ensure
+          flock LOCK_UN
+        end
+      else
+        raise SystemCallError, status
+    end
+  end
+end

data/lib/ghtorrent/command.rb

@@ -3,25 +3,44 @@ require 'trollop'
 require 'daemons'
 require 'etc'
 
-# Base class for all GHTorrent command line utilities. Provides basic command
-# line argument parsing and command bootstraping support. The order of
-# initialization is the following:
-# prepare_options
-# validate
-# go
+require 'ghtorrent/settings'
 
 module GHTorrent
+
+  # Base class for all GHTorrent command line utilities. Provides basic command
+  # line argument parsing and command bootstraping support. The order of
+  # initialization is the following:
+  # prepare_options
+  # validate
+  # go
   class Command
 
-    attr_reader :args, :options, :name
+    include GHTorrent::Settings
 
     # Specify the run method for subclasses.
     class << self
       def run(args = ARGV)
-        command = new(args)
+        attr_accessor :args
+        attr_accessor :settings
+        attr_accessor :name
+        attr_accessor :options
+
+        command = new()
+
+        command.name = self.class.name
+        command.args = args
+
         command.process_options
         command.validate
 
+        command.settings = YAML::load_file command.options[:config]
+
+        unless command.options[:addr].nil?
+          command.settings = command.override_config(command.settings,
+                                                     :attach_ip,
+                                                     command.options[:addr])
+        end
+
         if command.options[:daemon]
           if Process.uid == 0
             # Daemonize as a proper system daemon
@@ -59,15 +78,10 @@ module GHTorrent
       end
     end
 
-    def initialize(args)
-      @args = args
-      @name = self.class.name
-    end
-
-    # Specify and parse supported command line options.
+    # Specify and parse top-level command line options.
     def process_options
       command = self
-      @options = Trollop::options(@args) do
+      @options = Trollop::options(command.args) do
 
         command.prepare_options(self)
 
@@ -78,13 +92,12 @@ Standard options:
         opt :config, 'config.yaml file location', :short => 'c',
             :default => 'config.yaml'
         opt :verbose, 'verbose mode', :short => 'v'
+        opt :addr, 'ip address to use for performing requests', :short => 'a',
+            :type => String
         opt :daemon, 'run as daemon', :short => 'd'
         opt :user, 'run as the specified user (only when started as root)',
             :short => 'u', :type => String
       end
-
-      @args = @args.dup
-      ARGV.clear
     end
 
     # Get the version of the project
@@ -102,18 +115,19 @@ Standard options:
     # provided by this class.
     def validate
       if options[:config].nil?
-        unless (file_exists?("config.yaml") or file_exists?("/etc/ghtorrent/config.yaml"))
-          Trollop::die "No config file in default locations (., /etc/ghtorrent)
-                      you need to specify the #{:config} parameter. Read the
-                      documentation on how to create a config.yaml file."
+        unless (file_exists?("config.yaml"))
+          Trollop::die "No config file in default location (#{Dir.pwd}). You
+                        need to specify the #{:config} parameter. Read the
+                        documentation on how to create a config.yaml file."
         end
       else
-        Trollop::die "Cannot find file #{options[:config]}" unless file_exists?(options[:config])
+        Trollop::die "Cannot find file #{options[:config]}" \
+          unless file_exists?(options[:config])
       end
 
       unless @options[:user].nil?
         if not Process.uid == 0
-          Trollop::die "Option --user (-u) cannot be specified by normal users"
+          Trollop::die "Option --user (-u) can only be specified by root"
         end
           begin
             Etc.getpwnam(@options[:user])
@@ -132,6 +146,11 @@ Standard options:
     def go
     end
 
+    def override_config(config_file, setting, new_value)
+      STDERR.puts "Overriding configuration #{setting}=#{config(setting)} with cmd line #{new_value}"
+      merge_config_values({setting => new_value})
+    end
+
     private
 
     def file_exists?(file)
@@ -142,7 +161,6 @@ Standard options:
         false
       end
     end
-
   end
 
 end