gh-archive 0.9 → 0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 498649edc20ceb260d6468cd7e9ff46781e148205c17cc2c6dd65cd8c7258e8f
-  data.tar.gz: 6e14d05254a7dcddc3a5afdf1c5e749602fdffc508df70dd1a01ebc97642a78c
+  metadata.gz: 74e3037ee1115173176aa974f453f49b7649743f41f83118e9ee180bd620c095
+  data.tar.gz: c4c1ca30210ba39204b28b4b3854e2e93c69003ef1ead0f85506f2fe213f0ee9
 SHA512:
-  metadata.gz: 6b3156cdeec56577f61961b53c5552670534aaba0de3391daba5a99afc23fd09deb79da47879e2fc326eae4e85bbb180c41ca170540f67aa428c73571e443fad
-  data.tar.gz: 8129e09a30fd7abe02066ff890dcbd4d48f6965156f92d92582880883a11e508f397358aa39265fc65e7152c2e8204623a417c8689d3096c34e70df411e092f2
+  metadata.gz: 289b568dce07aa1f0182c75d26f7ab286a2b48dbf31c9ee63c6a1ef77bf5a59823b17136984380d6cca123c0be70e6cca3cc3dba216a935354d9312eb93aa2fa
+  data.tar.gz: 5d9ef4ec34a106e3fb2db37ab173c0b0876637941263f356f2e43440c506ad0471906cd4fa0fbd12bf98c84e44ee2bae0408a96ccdef0a5946c13487e5441204

data/lib/gh-archive.rb

@@ -9,6 +9,71 @@ require 'thread/promise'
 
 require_relative File.expand_path('../gh-archive/events', __FILE__)
 
+module GHArchive
+    class ThreadPool
+        def initialize(size)
+            @size = size
+            @threads = []
+            @queue = []
+            @mutex = Mutex.new
+            
+            @consumer_thread = Thread.start do
+                while !@shutdown || @threads.size > 0 || @queue.size > 0
+                    sleep 0.1 if @queue.size == 0 || @threads.size == @size
+                    @threads.delete_if { |t| !t.alive? }
+                    
+                    if @threads.size < @size && @queue.size > 0
+                        @mutex.synchronize do
+                            args, job = @queue.shift
+                            @threads << Thread.start(*args, &job)
+                        end
+                    end
+                end
+            end
+        end
+        
+        def process(*args, &block)
+            raise "Block expected" unless block_given?
+            raise "Can not add jobs while shutting down" if @shutdown
+            
+            @mutex.synchronize do
+                @queue << [args, block]
+            end
+            
+            return self.enqueued
+        end
+        
+        def shutdown
+            @shutdown = true
+        end
+        
+        def shutdown!
+            self.shutdown
+            @mutex.synchronize do
+                @queue.clear
+            end
+        end
+        
+        def enqueued
+            return @queue.size
+        end
+        
+        def shutdown?
+            @shutdown
+        end
+        
+        def alive?
+            @consumer_thread.alive?
+        end
+        
+        def wait
+            while alive?
+                sleep 0.1
+            end
+        end
+    end
+end
+
 module GHAUtils
     def get_gha_filename(date)
         return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
@@ -22,9 +87,10 @@ module GHAUtils
     end
     
     def read_gha_file(file)
-        if file.path.end_with?(".json")
+        
+        if !file.is_a?(StringIO) && file.path.end_with?(".json")
             content = file.read
-        elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
+        elsif file.is_a?(StringIO) || file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
             content = read_gha_file_content(file)
         else
             raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
@@ -101,9 +167,7 @@ class GHAProvider
         return self
     end
     
-    def each(from = Time.gm(2015, 1, 1), to = Time.now)
-        exceptions = []
-        
+    def restore_checkpoint(from)
         if @checkpoint_name && FileTest.exist?(@checkpoint_name)
             # Note that this throws an exception if the file is not readable. This is the intended behavior.
             # As opposed to that, failing to save the checkpoint information just results in a warning on the log.
@@ -111,11 +175,37 @@ class GHAProvider
             raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
             
             @logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
-            from = loaded_from
+            
+            return loaded_from
+        else
+            return from
         end
+    end
+    
+    def update_checkpoint(current_time)
+        if @checkpoint_name
+            begin
+                File.open(@checkpoint_name, "wb") do |f|
+                    f.write(Marshal.dump(current_time))
+                end
+            rescue
+                @logger.warn(
+                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                )
+            end
+        end
+    end
+    
+    def each(from = Time.gm(2015, 1, 1), to = Time.now)
+        exceptions = []
+        
+        from = restore_checkpoint(from)
         
         self.each_time(from, to) do |current_time|
             events = []
+            
+            update_checkpoint(current_time)
+            
             begin
                 events = self.get(current_time)
             rescue GHAException => e
@@ -127,18 +217,6 @@ class GHAProvider
                 next
             end
             
-            if @checkpoint_name
-                begin
-                    File.open(@checkpoint_name, "wb") do |f|
-                        f.write(Marshal.dump(current_time))
-                    end
-                rescue
-                    @logger.warn(
-                        "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
-                    )
-                end
-            end
-            
             events.each do |event|
                 skip = false
                 @includes.each do |key, value|
@@ -163,17 +241,7 @@ class GHAProvider
             GC.start
         end
         
-        if @checkpoint_name
-            begin
-                File.open(@checkpoint_name, "wb") do |f|
-                    f.write(Marshal.dump(to))
-                end
-            rescue
-                @logger.warn(
-                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
-                )
-            end
-        end
+        update_checkpoint(to)
         
         return exceptions
     end
@@ -186,13 +254,25 @@ class OnlineGHAProvider < GHAProvider
     def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
-        @max_retries = max_retries
-        @proactive = proactive
-        @proactive_pool_size = proactive_pool_size
-        @pool = Thread.pool(proactive_pool_size)
+        self.max_retries(max_retries)
+        self.proactive(proactive_pool_size) if proactive
+        
         @cache = Cache.new
     end
     
+    def max_retries(n)
+        @max_retries = n
+        
+        return self
+    end
+    
+    def proactive(pool_size = 10)
+        @proactive = true
+        @pool = GHArchive::ThreadPool.new(pool_size)
+        
+        return self
+    end
+    
     def get(current_time)        
         @max_retries.times do
             begin
@@ -252,16 +332,19 @@ class OnlineGHAProvider < GHAProvider
                 else
                     raise e
                 end
+            rescue Zlib::GzipFile::Error => e
+                @logger.warn("Could not unzip, cache and analyze the zip at #{current_time}: " + e.message)
             end
         end
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
+            real_from = restore_checkpoint(from)
             any_ready = Thread.promise
             
             @logger.info("Proactively scheduling download tasks...")
-            self.each_time(from, to) do |current_time|
+            self.each_time(real_from, to) do |current_time|
                 @pool.process(current_time) do |current_time|
                     cache(current_time)
                     any_ready << true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.9'
+  version: '0.13'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-14 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -63,7 +63,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []