ghtorrent 0.9 → 0.10

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MzBhZTlkNGYxYTc3ZTg4NzQ3OTQyMWQ3NTMxZGY1YzljYmE0MzIwZg==
-  data.tar.gz: !binary |-
-    NmE3OGNlNGY4Y2QzMGJkY2E3NWY0MTNhMTU2MDAyYTA3MDUzN2IzZQ==
+SHA1:
+  metadata.gz: 1394c5a3869bab8d02cf61fc8108ca1f10afc0b8
+  data.tar.gz: 9d82ecf68d0316db32f962af8cb7d0070dd74c82
 SHA512:
-  metadata.gz: !binary |-
-    MTkwZWQwNmQ2MjE1OTU5ZWQ5NWRmZWFhYjY4ZGM3NTA5ZmY2ZDcyMTI0OGZj
-    ZGY0MDczZGFmYTZlZTEyYmMxYWI4YTU5ODQyZGFmM2YwZmNhODljMDZmMWJk
-    ZThiYWQ5OGUxMmI3Yjg1Nzk1MjBkZTFiNzkxZmE1NTAzNGFmNzM=
-  data.tar.gz: !binary |-
-    MmJiODg3NmU2OTdjZTA2Njc0MmFmZWRhM2JkZjA4NjY5OTRiZTJhOTEzZGJk
-    NmJhN2JkNGFjODdhZDk0YWY4MDIzY2NkODU5MDhjZDQxNjA4NzU0ZWFkYmMx
-    ZDE2Y2U3NzQ0ZDAxN2YxNmNjZDRiZTUzMDQ5OWY5OWY3ZDc3Yjc=
+  metadata.gz: 7dbfe453c542137cafafa641c83bb680749e946c86193447e2b6aee829b88370bc66e90a8ea3d64a33d3c0061a7c5b01fa292ee96b94fba6dcca910d472d0f7f
+  data.tar.gz: d5c6487264b3a4d17af08e33ac2babd17b63ca6849e1ebd694a1d6b639797e8e870664f7dcde79718d61350551971a0c4b99b5d0ec56923b31984227c69b49bc

data/CHANGELOG

@@ -1,5 +1,20 @@
+= Version 0.10
+	* Base class for multiprocess queue clients
+	* Make retrieval of pull request commits faster
+	* Drop table forks
+	* Remove merged and user_id fields from pull_request
+	* Ruby 2.0.0 support
+	* Fast(er) path to project forking. Copy commits from base repo instead of getting them from Github.
+	* Github API tokens for authentication
+	* Support for disabling the cache
+	* A new real-time log analyzer
+	* Mark fake users as such
+	* Organization members are now retrieved
+	* Authenticated MongoDB connections
+	* General bug fixes and cleanups
+
 = Version 0.9
-    * Remove dependency to EventMachine-based AMQP client. We now use bunny.
+	* Remove dependency to EventMachine-based AMQP client. We now use bunny.
 
 = Version 0.8.1
 	* New tool to retrieve specific entities and their dependencies
@@ -8,7 +23,7 @@
 	* Support for finer grained transactions when processing large entities
 	* Commit comments are now indexed per owner/repo (was just by comment id)
 	* Remove the unused daemon mode
-    * Various exception fixes and more detailed logging
+	* Various exception fixes and more detailed logging
 
 = Version 0.8
 	* Retrieve and process issue labels

data/Gemfile.lock

@@ -1,28 +1,28 @@
 PATH
   remote: .
   specs:
-    ghtorrent (0.8.1)
-      bson_ext (~> 1.9.0)
-      bunny (~> 1.0.0)
-      mongo (~> 1.9.0)
-      sequel (~> 4.5.0)
-      trollop (~> 2.0.0)
+    ghtorrent (0.9)
+      bson_ext (~> 1.9, >= 1.9.0)
+      bunny (~> 1.0, >= 1.0.0)
+      mongo (~> 1.9, >= 1.9.0)
+      sequel (~> 4.5, >= 4.5.0)
+      trollop (~> 2.0, >= 2.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.3.5)
     amq-protocol (1.9.2)
-    bson (1.9.2)
-    bson_ext (1.9.2)
-      bson (~> 1.9.2)
-    bunny (1.0.7)
+    bson (1.10.0)
+    bson_ext (1.10.0)
+      bson (~> 1.10.0)
+    bunny (1.2.1)
       amq-protocol (>= 1.9.2)
     crack (0.4.1)
       safe_yaml (~> 0.9.0)
     diff-lcs (1.2.5)
-    mongo (1.9.2)
-      bson (~> 1.9.2)
+    mongo (1.10.0)
+      bson (~> 1.10.0)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
@@ -32,7 +32,7 @@ GEM
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
     safe_yaml (0.9.7)
-    sequel (4.5.0)
+    sequel (4.10.0)
     trollop (2.0)
     webmock (1.16.0)
       addressable (>= 2.2.7)
@@ -44,5 +44,5 @@ PLATFORMS
 DEPENDENCIES
   ghtorrent!
   jdbc-mysql
-  rspec (~> 2.14.0)
+  rspec (~> 2.14, >= 2.14.0)
   webmock (~> 1.16)

data/bin/ght-log-analyzer

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'pp'
+require 'time'
+
+stats = Hash.new { |hash, key| hash[key] = Hash.new }
+
+Thread.new do
+  puts "Collecting data..."
+  while (true) do
+    sleep(1)
+    system "clear" or system "cls"
+
+    stats.each do |k,v|
+      unless v[:time_in].nil?
+        v[:time_in] += 1
+      end
+    end
+
+    puts ' '
+    puts "ACTIVE"
+    puts ' '
+
+    data = stats.select { |k, v| v[:status] == 'A' }
+    ml = [:ip, :repo, :stage, :remaining, :not_found, :time_in].reduce({}) do |acc, x|
+      max = [data.map { |k, v| v[x].size }.max, x.to_s.length].max
+      acc.merge({x => max})
+    end
+
+    ml[:pid] = data.keys.map { |x| x.length }.max
+
+    fmt = "%-#{ml[:pid]}s %-#{ml[:ip]}s %-#{ml[:repo]}s %-#{ml[:stage]}s %-#{ml[:time_in]}s %-#{ml[:not_found]}s %-#{ml[:remaining]}s"
+
+    puts sprintf(fmt, 'pid', 'ip', 'repo', 'stage', 'time_in', 'not_found', 'remaining')
+    data.sort { |a, b| b[1][:time_in].to_i <=> a[1][:time_in].to_i }.each do |x|
+      k, v = x[0], x[1]
+      puts sprintf(fmt, k, v[:ip], v[:repo], v[:stage], v[:time_in], v[:not_found], v[:remaining])
+    end
+
+
+    puts ' '
+    puts "SLEEPING"
+    puts ' '
+
+    data = stats.select { |k, v| v[:status] == 'S' }
+    ml = [:ip, :repo, :stage, :sleep_remaining, :time_in].reduce({}) do |acc, x|
+      max = [data.map { |k, v|
+        if v[x].nil? then
+          1
+        else
+          v[x].size
+        end }.max, x.to_s.length].max
+        acc.merge({x => max})
+    end
+
+    ml[:pid] = data.keys.map { |x| x.length }.max
+
+    fmt = "%-#{ml[:pid]}s %-#{ml[:ip]}s %-#{ml[:repo]}s %-#{ml[:stage]}s %-#{ml[:time_in]}s %-#{ml[:sleep_remaining]}s"
+
+    puts sprintf(fmt, 'pid', 'ip', 'repo', 'stage', 'time_in', 'remaining_sleep')
+    data.sort { |a, b| a[1][:sleep_remaining].to_i <=> b[1][:sleep_remaining].to_i }.each do |x|
+      k, v = x[0], x[1]
+      puts sprintf(fmt, k, v[:ip], v[:repo], v[:stage], v[:time_in], v[:sleep_remaining])
+    end
+  end
+end
+
+
+ARGF.each do |x|
+
+  next unless x =~ /APIClient/
+
+  if x =~ /sleeping/
+    ts, pid, remaining = x.match(/\[([^.]+).*#([0-9]+)\].*for ([0-9]+).*/).captures
+
+    ts = Time.parse(ts).to_i
+    stats[pid][:status] = 'S'
+    stats[pid][:sleep_remaining] = remaining
+
+    unless stats[pid][:repo].nil?
+      stats[pid][:time_in] = ts - stats[pid][:start_pr_ts]
+    end
+
+  elsif x =~ /Not Found|Gone|Conflict/
+    pid = x.match(/.*#([0-9]+).*APIClient.*/).captures[0]
+    if stats[pid][:not_found].nil?
+      stats[pid][:not_found] = 0
+  end
+  stats[pid][:not_found] += 1
+  else
+    begin
+      ts, pid, ip, url, remaining, time =
+        x.match(/.*\[([^.]+).*#([0-9]+)\].*APIClient\[(.*)\].*(https:\/\/.*) \(([0-9]+) remaining\).* ([0-9]+) ms$/).captures
+    rescue
+      puts x
+      next
+    end
+    ts = Time.parse(ts).to_i
+    url_chunks = url.split(/\//)
+    owner = url_chunks[4]
+    repo = url_chunks[5]
+    begin
+      stage = url_chunks[6].split(/\?/)[0]
+    rescue
+      stage = 'repo'
+    end
+
+    stats[pid][:status] = 'A'
+    stats[pid][:ip] = ip
+
+    unless stats[pid][:repo] == "#{owner}/#{repo}"
+      stats[pid][:repo] = "#{owner}/#{repo}"
+      stats[pid][:not_found] = 0
+      stats[pid][:start_pr_ts] = ts
+      stats[pid][:time_in] = 0
+    end
+
+    stats[pid][:time_in] = ts - stats[pid][:start_pr_ts]
+    stats[pid][:stage] = stage
+    stats[pid][:remaining] = remaining
+
+    if stats[pid][:time].nil?
+      stats[pid][:time] = Queue.new
+    end
+
+    if stats[pid][:time].length > 100
+      stats[pid][:time].pop
+    end
+
+    stats[pid][:time].push(time)
+  end
+end
+

data/bin/ght-retrieve-one

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ghtorrent'
+
+GHTRetrieveOne.run(ARGV)

data/lib/ghtorrent.rb

@@ -50,6 +50,9 @@ require 'ghtorrent/retriever'
 require 'ghtorrent/ghtorrent'
 require 'ghtorrent/transacted_ghtorrent'
 
+# Multi-process queue clients
+require 'ghtorrent/multiprocess_queue_client'
+
 # Commands
 require 'ghtorrent/commands/ght_data_retrieval'
 require 'ghtorrent/commands/ght_mirror_events'
@@ -60,5 +63,6 @@ require 'ghtorrent/commands/ght_retrieve_repo'
 require 'ghtorrent/commands/ght_retrieve_user'
 require 'ghtorrent/commands/ght_retrieve_dependents'
 require 'ghtorrent/commands/ght_retrieve_repos'
+require 'ghtorrent/commands/ght_retrieve_one'
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/adapters/mongo_persister.rb

@@ -86,7 +86,7 @@ module GHTorrent
 
     def del(entity, query)
       super
-      raise Exception 'No filter was specifed. Cowardily refusing to remove all entries' if query == {}
+      raise Exception 'No filter was specified. Cowardily refusing to remove all entries' if query == {}
       get_entity(entity).remove(query)
     end
 
@@ -154,7 +154,7 @@ module GHTorrent
         @mongo = if replicas.nil?
                    Mongo::Connection.new(config(:mongo_host),
                                          config(:mongo_port))\
-                                    .db(config(:mongo_db))
+                                    .db(config(:mongo_db))\
                  else
                    repl_arr = replicas.strip.split(/ /).map{|x| "#{x}:#{config(:mongo_port)}"}
                    repl_arr << "#{config(:mongo_host)}:#{config(:mongo_port)}"
@@ -162,6 +162,7 @@ module GHTorrent
                                            .db(config(:mongo_db))
                  end
 
+        @mongo.authenticate(config(:mongo_username), config(:mongo_passwd))
         stats = @mongo.stats
         init_db(@mongo) if stats['collections'] < ENTITIES.size + 2
         init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size

data/lib/ghtorrent/api_client.rb

@@ -28,26 +28,8 @@ module GHTorrent
     def paged_api_request(url, pages = config(:mirror_history_pages_back),
         cache = true, last = nil)
 
-      url = if not url.include?('per_page')
-              if url.include?('?')
-                url + '&per_page=100'
-              else
-                url + '?per_page=100'
-              end
-            else
-              url
-            end
-
-      params = CGI::parse(URI::parse(url).query)
-      data = if params.has_key?('page') or (params.has_key?('last_sha'))
-               api_request_raw(url, use_cache?(cache, method = :paged))
-             else
-               if @cache_mode == :all
-                 api_request_raw(url, true)
-               else
-                 api_request_raw(url, false)
-              end
-             end
+      url = ensure_max_per_page(url)
+      data = determine_cache_and_do_request(cache, url)
 
       return [] if data.nil?
 
@@ -80,14 +62,64 @@ module GHTorrent
       end
     end
 
+
     # A normal request. Returns a hash or an array of hashes representing the
     # parsed JSON result.
     def api_request(url, cache = true)
-      parse_request_result api_request_raw(url, use_cache?(cache))
+      parse_request_result api_request_raw(ensure_max_per_page(url), use_cache?(cache))
+    end
+
+    # Determine the number of pages contained in a multi-page API response
+    def num_pages(url)
+      url = ensure_max_per_page(url)
+      data = determine_cache_and_do_request(true, url)
+
+      if data.meta.nil? or data.meta['link'].nil?
+        return 1
+      end
+
+      links = parse_links(data.meta['link'])
+
+      if links.nil? or links['last'].nil?
+        return 1
+      end
+
+      params = CGI::parse(URI::parse(links['last']).query)
+      params['page'][0].to_i
     end
 
     private
 
+    def ensure_max_per_page(url)
+      if url.include?('page')
+        if not url.include?('per_page')
+          if url.include?('?')
+            url + '&per_page=100'
+          else
+            url + '?per_page=100'
+          end
+        else
+          url
+        end
+      else
+        url
+      end
+    end
+
+    def determine_cache_and_do_request(cache, url)
+      query = URI::parse(url).query
+      params = unless query.nil?
+                 CGI::parse(query)
+               else
+                 {}
+               end
+      if params.has_key?('page') or (params.has_key?('last_sha'))
+        api_request_raw(url, use_cache?(cache, method = :paged))
+      else
+        api_request_raw(url, use_cache?(cache, method = :non_paged))
+      end
+    end
+
     # Determine whether to use cache or not, depending on the type of the
     # request
     def use_cache?(client_request, method = :non_paged)
@@ -98,10 +130,15 @@ module GHTorrent
                           :prod
                         when 'all'
                           :all
+                        when 'off'
+                        when false
+                          :off
                         else
-                          raise GHTorrentException.new("Don't know cache configuration #{@cache_mode}")
+                          raise GHTorrentException.new("Don't know cache configuration #{config(:cache_mode)}")
                       end
       case @cache_mode
+        when :off
+          return false
         when :all
           return true
         when :dev
@@ -180,7 +217,8 @@ module GHTorrent
               403, # Forbidden
               404, # Not found
               422 then # Unprocessable entity
-            warn "APIClient: #{url}: #{e.io.status[1]}"
+            total = Time.now.to_ms - start_time.to_ms
+            warn "APIClient[#{@attach_ip}]: Request: #{url} (#{@remaining} remaining), Total: #{total} ms, Status: #{e.io.status[1]}"
             @remaining = e.io.meta['x-ratelimit-remaining'].to_i
             @reset = e.io.meta['x-ratelimit-reset'].to_i
             return nil
@@ -191,27 +229,55 @@ module GHTorrent
       ensure
         if not from_cache and config(:respect_api_ratelimit) and @remaining < 10
           to_sleep = @reset - Time.now.to_i + 2
-          debug "APIClient: Request limit reached, sleeping for #{to_sleep} secs"
+          debug "APIClient[#{@attach_ip}]: Request limit reached, sleeping for #{to_sleep} secs"
+          t = Thread.new do
+            slept = 0
+            while true do
+              debug "APIClient[#{@attach_ip}]: sleeping for #{to_sleep - slept} seconds"
+              sleep 1
+              slept += 1
+            end
+          end
           sleep(to_sleep)
+          t.exit
+        end
+      end
+    end
+
+    def auth_method(username, token)
+      if token.nil? or token.empty?
+        if username.nil? or username.empty?
+          :none
+        else
+          :username
         end
+      else
+        :token
       end
     end
 
     def do_request(url)
       @attach_ip  ||= config(:attach_ip)
+      @token      ||= config(:github_token)
       @username   ||= config(:github_username)
       @passwd     ||= config(:github_passwd)
       @user_agent ||= config(:user_agent)
       @remaining  ||= 10
       @reset      ||= Time.now.to_i + 3600
-
-      open_func ||= if @username.nil?
-        lambda {|url| open(url, 'User-Agent' => @user_agent)}
-      else
-        lambda {|url| open(url,
-                           'User-Agent' => @user_agent,
-                           :http_basic_authentication => [@username, @passwd])}
-      end
+      @auth_type  ||= auth_method(@username, @token)
+
+      open_func ||=
+          case @auth_type
+            when :none
+              lambda {|url| open(url, 'User-Agent' => @user_agent)}
+            when :username
+              lambda {|url| open(url, 'User-Agent' => @user_agent,
+                                 :http_basic_authentication => [@username, @passwd])}
+            when :token
+              # As per: https://developer.github.com/v3/auth/#via-oauth-tokens
+              lambda {|url| open(url, 'User-Agent' => @user_agent,
+                                 :http_basic_authentication => [@token, 'x-oauth-basic'])}
+          end
 
       result = if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0'
           open_func.call(url)
@@ -231,8 +297,15 @@ module GHTorrent
         (class << self; self; end).instance_eval do
           alias_method :original_open, :open
 
-          define_method(:open) do |conn_address, conn_port|
-            original_open(conn_address, conn_port, ip)
+          case RUBY_VERSION
+          when /1.9/
+            define_method(:open) do |conn_address, conn_port|
+              original_open(conn_address, conn_port, ip)
+            end
+          when /2.0/
+            define_method(:open) do |conn_address, conn_port, local_host, local_port|
+              original_open(conn_address, conn_port, ip, local_port)
+            end
           end
         end
       end