gh-archive 0.8 → 0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4a9b89fb02620499e8b51f5c5e4ba34d0f3bc8b8f3ae4e2e69cba1e027bdb49
-  data.tar.gz: 6b707eb1bcb37b8a9b03ce36a2d4304e6760def0cc139ffad748a1596046f82f
+  metadata.gz: 9e36482fd8eeb76b12db28c1e68ad836d77a5985af6aa0e570a025bcb664b1a0
+  data.tar.gz: a91e869a8e3f614e8280f03749bcd31ce0de3c7c4e9dc9dd3d777c1a3f1e0d3f
 SHA512:
-  metadata.gz: '0975e354e028e768fb5bc4c17c19cddbc44b706394f60d8e22a64537a34342965bb98fa00365c329c30d189729dadba271ec6b101ff86593affef7e1d34848b3'
-  data.tar.gz: 95935fb3c27841760a68696c832e02108940f9eef535f5ed89afea9613a7988c622f2649f3feb77b22706250a30cfba30b5f36f2ada617f532e86a0426e052ce
+  metadata.gz: 4423afb5e0538be2abbe4ac21aeba3919081f0c61943ad9801add90b85b6f2e9df907836c13e00c6b300b28766c65c5cb2e78bc040393ebd38c9618092f2957d
+  data.tar.gz: c0caf8f4e47419744f7694748ed608c2e7da8fc0094477ce96bad211cf0b398ef5d43550fdcfd25ba1bc820cafd470ebc4f7526c39c171ef991bfc5fd2399882

data/lib/gh-archive.rb

@@ -9,6 +9,71 @@ require 'thread/promise'
 
 require_relative File.expand_path('../gh-archive/events', __FILE__)
 
+module GHArchive
+    class ThreadPool
+        def initialize(size)
+            @size = size
+            @threads = []
+            @queue = []
+            @mutex = Mutex.new
+            
+            @consumer_thread = Thread.start do
+                while !@shutdown || @threads.size > 0 || @queue.size > 0
+                    sleep 0.1 if @queue.size == 0 || @threads.size == @size
+                    @threads.delete_if { |t| !t.alive? }
+                    
+                    if @threads.size < @size && @queue.size > 0
+                        @mutex.synchronize do
+                            args, job = @queue.shift
+                            @threads << Thread.start(*args, &job)
+                        end
+                    end
+                end
+            end
+        end
+        
+        def process(*args, &block)
+            raise "Block expected" unless block_given?
+            raise "Can not add jobs while shutting down" if @shutdown
+            
+            @mutex.synchronize do
+                @queue << [args, block]
+            end
+            
+            return self.enqueued
+        end
+        
+        def shutdown
+            @shutdown = true
+        end
+        
+        def shutdown!
+            self.shutdown
+            @mutex.synchronize do
+                @queue.clear
+            end
+        end
+        
+        def enqueued
+            return @queue.size
+        end
+        
+        def shutdown?
+            @shutdown
+        end
+        
+        def alive?
+            @consumer_thread.alive?
+        end
+        
+        def wait
+            while alive?
+                sleep 0.1
+            end
+        end
+    end
+end
+
 module GHAUtils
     def get_gha_filename(date)
         return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
@@ -22,9 +87,10 @@ module GHAUtils
     end
     
     def read_gha_file(file)
-        if file.path.end_with?(".json")
+        
+        if !file.is_a?(StringIO) && file.path.end_with?(".json")
             content = file.read
-        elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
+        elsif file.is_a?(StringIO) || file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
             content = read_gha_file_content(file)
         else
             raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
@@ -56,9 +122,16 @@ class GHAProvider
         @includes = {}
         @excludes = {}
         
+        @checkpoint_name = nil
         @use_json = true
     end
     
+    def use_checkpoint(filename)
+        @checkpoint_name = filename
+        
+        return self
+    end
+    
     def parse_events
         @use_json = false
         
@@ -67,7 +140,10 @@ class GHAProvider
     
     def logger=(logger)
         @logger = logger
+        
+        return self
     end
+    alias :use_logger :logger=
     
     def get(date)
         raise "Not implemented"
@@ -91,11 +167,45 @@ class GHAProvider
         return self
     end
     
+    def restore_checkpoint(from)
+        if @checkpoint_name && FileTest.exist?(@checkpoint_name)
+            # Note that this throws an exception if the file is not readable. This is the intended behavior.
+            # As opposed to that, failing to save the checkpoint information just results in a warning on the log.
+            loaded_from = Marshal.load(File.read(@checkpoint_name))
+            raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
+            
+            @logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
+            
+            return loaded_from
+        else
+            return from
+        end
+    end
+    
+    def update_checkpoint(current_time)
+        if @checkpoint_name
+            begin
+                File.open(@checkpoint_name, "wb") do |f|
+                    f.write(Marshal.dump(current_time))
+                end
+            rescue
+                @logger.warn(
+                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                )
+            end
+        end
+    end
+    
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         exceptions = []
         
+        from = restore_checkpoint(from)
+        
         self.each_time(from, to) do |current_time|
             events = []
+            
+            update_checkpoint(current_time)
+            
             begin
                 events = self.get(current_time)
             rescue GHAException => e
@@ -131,6 +241,8 @@ class GHAProvider
             GC.start
         end
         
+        update_checkpoint(to)
+        
         return exceptions
     end
     
@@ -142,13 +254,25 @@ class OnlineGHAProvider < GHAProvider
     def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
-        @max_retries = max_retries
-        @proactive = proactive
-        @proactive_pool_size = proactive_pool_size
-        @pool = Thread.pool(proactive_pool_size)
+        self.max_retries(max_retries)
+        self.proactive(proactive_pool_size) if proactive
+        
         @cache = Cache.new
     end
     
+    def max_retries(n)
+        @max_retries = n
+        
+        return self
+    end
+    
+    def proactive(pool_size = 10)
+        @proactive = true
+        @pool = GHArchive::ThreadPool.new(pool_size)
+        
+        return self
+    end
+    
     def get(current_time)        
         @max_retries.times do
             begin
@@ -214,10 +338,11 @@ class OnlineGHAProvider < GHAProvider
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
+            real_from = restore_checkpoint(from)
             any_ready = Thread.promise
             
             @logger.info("Proactively scheduling download tasks...")
-            self.each_time(from, to) do |current_time|
+            self.each_time(real_from, to) do |current_time|
                 @pool.process(current_time) do |current_time|
                     cache(current_time)
                     any_ready << true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.8'
+  version: '0.12'
 platform: ruby
 authors:
 - Simone Scalabrino
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-14 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions