gh-archive 0.10 → 0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1a8f0a5088aa3d598466d5ef3ba3472b5e9b0c2ad3327f291c24045c01bdfba
-  data.tar.gz: b3df61810f7634eb8e9a153a812fa2073078975ee0ca303bfb9fb87e58f08160
+  metadata.gz: '09dad47f83d5fe7255be12f2aaa380331de9fc34380bc9c02d6288d7216c7778'
+  data.tar.gz: c65569657989ad1df63e16e0ca50dfb91c97674465a767099b8bcb207aad8d16
 SHA512:
-  metadata.gz: f5462029f6ef8e32f632bfdb7552606cf47a924e019502288ee2d841e28e76a5899d944d912a41933834e08e7a813c88cc919a7e93dba42b1c3250fcd911ae6e
-  data.tar.gz: 1f180c6640e38423ac2cde87ee4181cf04db8bb0a5cc6f861fd39d5de0c5bc4d30df87524f594482c170d591c5cd063abfa539b5715a4850056ab5262acef965
+  metadata.gz: 5c4ad892837193a4665bb22241480db59bb1a3757b0cbde72cfef6e4ff242006c60591a83183b02215e6e05aa3efaa79d2ce71d6e802dbb5b5d210a1f0b462f7
+  data.tar.gz: 24161291f01c2fb86f367f7c3b03045e4a8853433d3a4d0075a131adf2dba65f337808b5cac9480d131c9afae4b346c5904d99f630b1aa8115bb4a285832aedf

data/lib/gh-archive.rb

@@ -9,6 +9,71 @@ require 'thread/promise'
 
 require_relative File.expand_path('../gh-archive/events', __FILE__)
 
+module GHArchive
+    class ThreadPool
+        def initialize(size)
+            @size = size
+            @threads = []
+            @queue = []
+            @mutex = Mutex.new
+            
+            @consumer_thread = Thread.start do
+                while !@shutdown || @threads.size > 0 || @queue.size > 0
+                    sleep 0.1 if @queue.size == 0 || @threads.size == @size
+                    @threads.delete_if { |t| !t.alive? }
+                    
+                    if @threads.size < @size && @queue.size > 0
+                        @mutex.synchronize do
+                            args, job = @queue.shift
+                            @threads << Thread.start(*args, &job)
+                        end
+                    end
+                end
+            end
+        end
+        
+        def process(*args, &block)
+            raise "Block expected" unless block_given?
+            raise "Can not add jobs while shutting down" if @shutdown
+            
+            @mutex.synchronize do
+                @queue << [args, block]
+            end
+            
+            return self.enqueued
+        end
+        
+        def shutdown
+            @shutdown = true
+        end
+        
+        def shutdown!
+            self.shutdown
+            @mutex.synchronize do
+                @queue.clear
+            end
+        end
+        
+        def enqueued
+            return @queue.size
+        end
+        
+        def shutdown?
+            @shutdown
+        end
+        
+        def alive?
+            @consumer_thread.alive?
+        end
+        
+        def wait
+            while alive?
+                sleep 0.1
+            end
+        end
+    end
+end
+
 module GHAUtils
     def get_gha_filename(date)
         return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
@@ -22,9 +87,10 @@ module GHAUtils
     end
     
     def read_gha_file(file)
-        if file.path.end_with?(".json")
+        
+        if !file.is_a?(StringIO) && file.path.end_with?(".json")
             content = file.read
-        elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
+        elsif file.is_a?(StringIO) || file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
             content = read_gha_file_content(file)
         else
             raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
@@ -137,6 +203,9 @@ class GHAProvider
         
         self.each_time(from, to) do |current_time|
             events = []
+            
+            update_checkpoint(current_time)
+            
             begin
                 events = self.get(current_time)
             rescue GHAException => e
@@ -148,8 +217,6 @@ class GHAProvider
                 next
             end
             
-            update_checkpoint(current_time)
-            
             events.each do |event|
                 skip = false
                 @includes.each do |key, value|
@@ -187,13 +254,25 @@ class OnlineGHAProvider < GHAProvider
     def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
-        @max_retries = max_retries
-        @proactive = proactive
-        @proactive_pool_size = proactive_pool_size
-        @pool = Thread.pool(proactive_pool_size)
+        self.max_retries(max_retries)
+        self.proactive(proactive_pool_size) if proactive
+        
         @cache = Cache.new
     end
     
+    def max_retries(n)
+        @max_retries = n
+        
+        return self
+    end
+    
+    def proactive(pool_size = 10)
+        @proactive = true
+        @pool = GHArchive::ThreadPool.new(pool_size)
+        
+        return self
+    end
+    
     def get(current_time)        
         @max_retries.times do
             begin
@@ -250,9 +329,13 @@ class OnlineGHAProvider < GHAProvider
                 if code.start_with?("5")
                     @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
                     next
+                elsif code == "404"
+                    @logger.error("File for #{current_time} not found. Skipping because: " + e.message)
                 else
                     raise e
                 end
+            rescue Zlib::GzipFile::Error => e
+                @logger.warn("Could not unzip, cache and analyze the zip at #{current_time}: " + e.message)
             end
         end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.10'
+  version: '0.14'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -63,7 +63,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []