ghtorrent 0.10 → 0.11

data/bin/ght-queue-grep.rb

@@ -0,0 +1,55 @@
+require 'json'
+require 'ghtorrent'
+
+
+class GHTQueueGrep < GHTorrent::Command
+
+  include GHTorrent::Persister
+
+  def persister
+    @persister ||= connect(:mongo, settings)
+    @persister
+  end
+
+  def retrieve_event(evt_id)
+    event = persister.get_underlying_connection[:events].find_one('id' => evt_id)
+    event.delete '_id'
+    data = JSON.parse(event.to_json)
+    data
+  end
+
+  def go
+
+    processor = Proc.new do |evt_id|
+      e = retrieve_event(evt_id)
+
+      comment = case e['type']
+                  when /CommitCommentEvent/
+                    e['payload']['comment']['body']
+                  when /PullRequestReviewCommentEvent/
+                    e['payload']['body']
+                  when /IssueCommentEvent/
+                    e['payload']['comment']['body']
+                end
+
+      url = case e['type']
+              when /CommitCommentEvent/
+                e['payload']['comment']['html_url']
+              when /PullRequestReviewCommentEvent/
+                e['payload']['comment']['html_url']
+              when /IssueCommentEvent/
+                e['payload']['comment']['html_url']
+            end
+
+      if comment =~ %r[security|csrf|xss|injection]im
+        puts "#{e['repo']['name']} #{url}"
+      end
+
+    end
+
+    queue_client 'comments', processor
+  end
+
+end
+
+GHTQueueGrep.run

data/bin/ght-retrieve-users

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ghtorrent'
+
+GHTRetrieveUsers.run(ARGV)

data/bin/{ght-rm-dupl → ght-update-repo}

@@ -3,4 +3,4 @@
 require 'rubygems'
 require 'ghtorrent'
 
-GHRMDupl.run
+GHTUpdateRepo.run(ARGV)

data/lib/ghtorrent.rb

@@ -27,11 +27,9 @@ require 'ghtorrent/bson_orderedhash'
 
 # Basic utility modules
 require 'version'
-require 'ghtorrent/gh_torrent_exception'
 require 'ghtorrent/utils'
 require 'ghtorrent/logging'
 require 'ghtorrent/settings'
-require 'ghtorrent/cache'
 require 'ghtorrent/api_client'
 
 # Support for command line utilities offered by this gem
@@ -47,8 +45,9 @@ require 'ghtorrent/persister'
 require 'ghtorrent/retriever'
 
 # SQL database fillup methods
+require 'ghtorrent/event_processing'
 require 'ghtorrent/ghtorrent'
-require 'ghtorrent/transacted_ghtorrent'
+require 'ghtorrent/transacted_gh_torrent'
 
 # Multi-process queue clients
 require 'ghtorrent/multiprocess_queue_client'
@@ -57,12 +56,13 @@ require 'ghtorrent/multiprocess_queue_client'
 require 'ghtorrent/commands/ght_data_retrieval'
 require 'ghtorrent/commands/ght_mirror_events'
 require 'ghtorrent/commands/ght_get_more_commits'
-require 'ghtorrent/commands/ght_rm_dupl'
 require 'ghtorrent/commands/ght_load'
 require 'ghtorrent/commands/ght_retrieve_repo'
 require 'ghtorrent/commands/ght_retrieve_user'
 require 'ghtorrent/commands/ght_retrieve_dependents'
 require 'ghtorrent/commands/ght_retrieve_repos'
 require 'ghtorrent/commands/ght_retrieve_one'
+require 'ghtorrent/commands/ght_retrieve_users'
+require 'ghtorrent/commands/ght_update_repo'
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/adapters/base_adapter.rb

@@ -11,7 +11,7 @@ module GHTorrent
     # Stores +data+ into +entity+. Returns a unique key for the stored entry.
     def store(entity, data = {})
       unless ENTITIES.include?(entity)
-        raise GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise "Perister: Entity #{entity} not known"
       end
     end
 
@@ -52,14 +52,7 @@ module GHTorrent
     # matching JSON object.
     def find(entity, query = {})
       unless ENTITIES.include?(entity)
-        raise GHTorrentException.new("Perister: Entity #{entity} not known")
-      end
-    end
-
-    # Find the record identified by +id+ in +entity+
-    def find_by_ext_ref_id(entity, id)
-      unless ENTITIES.include?(entity)
-        raise GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise "Perister: Entity #{entity} not known"
       end
     end
 
@@ -67,13 +60,13 @@ module GHTorrent
     # The +query+ can be any query supported by +find+.
     def count(entity, query = {})
       unless ENTITIES.include?(entity)
-        raise GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise "Perister: Entity #{entity} not known"
       end
     end
 
     def del(entity, query = {})
       unless ENTITIES.include?(entity)
-        raise GHTorrentException.new("Perister: Entity #{entity} not known")
+        raise "Perister: Entity #{entity} not known"
       end
     end
 

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -70,12 +70,6 @@ module GHTorrent
       }
     end
 
-    # Find the record identified by +id+ in +entity+
-    def find_by_ext_ref_id(entity, id)
-      super
-      raise NotImplementedError
-    end
-
     # Count the number of items returned by +query+
     def count(entity, query)
       super
@@ -162,10 +156,12 @@ module GHTorrent
                                            .db(config(:mongo_db))
                  end
 
-        @mongo.authenticate(config(:mongo_username), config(:mongo_passwd))
+        unless config(:mongo_username).nil?
+          @mongo.authenticate(config(:mongo_username), config(:mongo_passwd))
+        end
         stats = @mongo.stats
-        init_db(@mongo) if stats['collections'] < ENTITIES.size + 2
-        init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size
+        #init_db(@mongo) if stats['collections'] < ENTITIES.size + 2
+        #init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size
 
         @mongo
       else

data/lib/ghtorrent/adapters/noop_persister.rb

@@ -18,11 +18,6 @@ module GHTorrent
       []
     end
 
-    def find_by_ext_ref_id(entity, id)
-      super
-      nil
-    end
-
     def get_id
       0
     end

data/lib/ghtorrent/api_client.rb

@@ -7,14 +7,12 @@ require 'json'
 require 'ghtorrent/logging'
 require 'ghtorrent/settings'
 require 'ghtorrent/time'
-require 'ghtorrent/cache'
 require 'version'
 
 module GHTorrent
   module APIClient
     include GHTorrent::Logging
     include GHTorrent::Settings
-    include GHTorrent::Cache
     include GHTorrent::Logging
 
     # This is to fix an annoying bug in JRuby's SSL not being able to
@@ -26,10 +24,10 @@ module GHTorrent
     # A paged request. Used when the result can expand to more than one
     # result pages.
     def paged_api_request(url, pages = config(:mirror_history_pages_back),
-        cache = true, last = nil)
+                          last = nil)
 
       url = ensure_max_per_page(url)
-      data = determine_cache_and_do_request(cache, url)
+      data = api_request_raw(url)
 
       return [] if data.nil?
 
@@ -47,15 +45,7 @@ module GHTorrent
         if links['next'].nil?
           parse_request_result(data)
         else
-          parse_request_result(data) |
-              if links['next'] == last
-                if last != links['last']
-                  warn "APIClient: Last header mismatch: method=#{last}, cache=#{links['last']}"
-                end
-                paged_api_request(links['next'], pages, false, last)
-              else
-                paged_api_request(links['next'], pages, cache, last)
-              end
+          parse_request_result(data) | paged_api_request(links['next'], pages, last)
         end
       else
         parse_request_result(data)
@@ -65,16 +55,16 @@ module GHTorrent
 
     # A normal request. Returns a hash or an array of hashes representing the
     # parsed JSON result.
-    def api_request(url, cache = true)
-      parse_request_result api_request_raw(ensure_max_per_page(url), use_cache?(cache))
+    def api_request(url)
+      parse_request_result api_request_raw(ensure_max_per_page(url))
     end
 
     # Determine the number of pages contained in a multi-page API response
     def num_pages(url)
       url = ensure_max_per_page(url)
-      data = determine_cache_and_do_request(true, url)
+      data = api_request_raw(url)
 
-      if data.meta.nil? or data.meta['link'].nil?
+      if data.nil? or data.meta.nil? or data.meta['link'].nil?
         return 1
       end
 
@@ -106,60 +96,6 @@ module GHTorrent
       end
     end
 
-    def determine_cache_and_do_request(cache, url)
-      query = URI::parse(url).query
-      params = unless query.nil?
-                 CGI::parse(query)
-               else
-                 {}
-               end
-      if params.has_key?('page') or (params.has_key?('last_sha'))
-        api_request_raw(url, use_cache?(cache, method = :paged))
-      else
-        api_request_raw(url, use_cache?(cache, method = :non_paged))
-      end
-    end
-
-    # Determine whether to use cache or not, depending on the type of the
-    # request
-    def use_cache?(client_request, method = :non_paged)
-      @cache_mode ||= case config(:cache_mode)
-                        when 'dev'
-                          :dev
-                        when 'prod'
-                          :prod
-                        when 'all'
-                          :all
-                        when 'off'
-                        when false
-                          :off
-                        else
-                          raise GHTorrentException.new("Don't know cache configuration #{config(:cache_mode)}")
-                      end
-      case @cache_mode
-        when :off
-          return false
-        when :all
-          return true
-        when :dev
-          unless client_request
-            return false
-          end
-          return true
-        when :prod
-          if client_request
-            return true
-          else
-            case method
-              when :non_paged
-                return false
-              when :paged
-                return true
-            end
-          end
-      end
-    end
-
     # Parse a Github link header
     def parse_links(links)
       links.split(/,/).reduce({}) do |acc, x|
@@ -184,33 +120,39 @@ module GHTorrent
       end
     end
 
+    def request_error_msg(url, exception)
+      <<-MSG
+            Failed request. URL: #{url}, Status code: #{exception.io.status[0]},
+            Status: #{exception.io.status[1]},
+            Access: #{if (@token.nil? or @token.empty?) then @username else @token end},
+            IP: #{@attach_ip}, Remaining: #{@remaining}
+      MSG
+    end
+
+    def error_msg(url, exception)
+      <<-MSG
+            Failed request. URL: #{url}, Exception: #{exception.message},
+            Access: #{if (@token.nil? or @token.empty?) then @username else @token end},
+            IP: #{@attach_ip}, Remaining: #{@remaining}
+      MSG
+    end
+
     # Do the actual request and return the result object
-    def api_request_raw(url, use_cache = false)
+    def api_request_raw(url)
 
       begin
         start_time = Time.now
-        from_cache = false
-
-        contents =
-            if use_cache
-              if not (cached = cache_get(url)).nil?
-                from_cache = true
-                cached
-              else
-                tocache = Cachable.new(do_request(url))
-                cache_put(url, tocache)
-                tocache
-              end
-            else
-              do_request(url)
-            end
 
+        contents = do_request(url)
         total = Time.now.to_ms - start_time.to_ms
-        debug "APIClient[#{@attach_ip}]: Request: #{url} #{if from_cache then "from cache," else "(#{@remaining} remaining)," end} Total: #{total} ms"
+        info "Successful request. URL: #{url}, Remaining: #{@remaining}, Total: #{total} ms"
 
         contents
       rescue OpenURI::HTTPError => e
-        case e.io.status[0].to_i
+          @remaining = e.io.meta['x-ratelimit-remaining'].to_i
+          @reset = e.io.meta['x-ratelimit-reset'].to_i
+
+          case e.io.status[0].to_i
           # The following indicate valid Github return codes
           when 400, # Bad request
               401, # Unauthorized
@@ -218,22 +160,24 @@ module GHTorrent
               404, # Not found
               422 then # Unprocessable entity
             total = Time.now.to_ms - start_time.to_ms
-            warn "APIClient[#{@attach_ip}]: Request: #{url} (#{@remaining} remaining), Total: #{total} ms, Status: #{e.io.status[1]}"
-            @remaining = e.io.meta['x-ratelimit-remaining'].to_i
-            @reset = e.io.meta['x-ratelimit-reset'].to_i
-            return nil
+            warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
+           return nil
           else # Server error or HTTP conditions that Github does not report
-            warn "APIClient: #{url}: #{e.io.status[1]}"
+            warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
             raise e
         end
+      rescue StandardError => e
+        warn error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
+        raise e
       ensure
-        if not from_cache and config(:respect_api_ratelimit) and @remaining < 10
+        # The exact limit is only enforced upon the first @reset
+        if 5000 - @remaining >= @req_limit
           to_sleep = @reset - Time.now.to_i + 2
-          debug "APIClient[#{@attach_ip}]: Request limit reached, sleeping for #{to_sleep} secs"
+          debug "Request limit reached, sleeping for #{to_sleep} secs"
           t = Thread.new do
             slept = 0
             while true do
-              debug "APIClient[#{@attach_ip}]: sleeping for #{to_sleep - slept} seconds"
+              debug "Sleeping for #{to_sleep - slept} seconds"
               sleep 1
               slept += 1
             end
@@ -262,9 +206,10 @@ module GHTorrent
       @username   ||= config(:github_username)
       @passwd     ||= config(:github_passwd)
       @user_agent ||= config(:user_agent)
-      @remaining  ||= 10
+      @remaining  ||= 5000
       @reset      ||= Time.now.to_i + 3600
       @auth_type  ||= auth_method(@username, @token)
+      @req_limit  ||= config(:req_limit)
 
       open_func ||=
           case @auth_type
@@ -276,7 +221,7 @@ module GHTorrent
             when :token
               # As per: https://developer.github.com/v3/auth/#via-oauth-tokens
               lambda {|url| open(url, 'User-Agent' => @user_agent,
-                                 :http_basic_authentication => [@token, 'x-oauth-basic'])}
+                                 'Authorization' => "token #{@token}") }
           end
 
       result = if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0'
@@ -312,7 +257,7 @@ module GHTorrent
 
       result = begin
         yield
-      rescue Exception => e
+      rescue StandardError => e
         raise e
       ensure
         TCPSocket.instance_eval do
@@ -328,22 +273,3 @@ module GHTorrent
 
   end
 end
-
-class Cachable
-
-  include OpenURI::Meta
-
-  attr_reader :base_uri, :meta, :status
-
-  def initialize(response)
-    @data = response.read
-    @base_uri = response.base_uri
-    @meta = response.meta
-    @status = response.status
-  end
-
-  def read
-    @data
-  end
-
-end