gh-archive 0.7 → 0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40425d8038835d1e22dc684d3eb1570d06524cd2eb2216a5a6b140a33636da23
-  data.tar.gz: a188da9ef8eb14116ca6abbdfdd933fa01cbe5900ad2e2574108e9ed0223e7af
+  metadata.gz: 652cb53430d2e6a230d8f11465b81685f2e92107868345566d16b07d4c4231ed
+  data.tar.gz: abaec255c079e5a959fd51f7b1e7ca15d72c21eaaeac98d34d13309ec91eedeb
 SHA512:
-  metadata.gz: 19edf6c81e859d32a188e433dcba0f0e7cf76da43c7b65245ea9c97ef6f419add6a30192701841e82a5203856998f2905f0d6ee677027d78e32b68c64190dbe1
-  data.tar.gz: c8e7aecf5cfb326e2a0e09c4ceed3e148db4b7520252c2bcecfedb008d86726a31ad33fad944de0b8eadeff0f73ec552e5e71d98add6ce692f9a81b326e9631c
+  metadata.gz: 829b1d53a7b72f4ea83712e8a95ba059e48dff7c982e811d30d27018e45e1b7010ace2a4a3f2641318a598e765a6d2c6138b319f9d67dd2604617dc2785175be
+  data.tar.gz: fd85da1c6630e8b534fb34fc9b5a08fcda55e3d2606ca393157134b6d85da2a9a7f92bc87ae31369441da1799cfd2178f0af9703ca997d10296c3f9a5ab1e4f9

data/lib/gh-archive.rb

@@ -7,7 +7,7 @@ require 'tmpdir'
 require 'thread/pool'
 require 'thread/promise'
 
-require 'gh-archive/events'
+require_relative File.expand_path('../gh-archive/events', __FILE__)
 
 module GHAUtils
     def get_gha_filename(date)
@@ -56,9 +56,16 @@ class GHAProvider
         @includes = {}
         @excludes = {}
         
+        @checkpoint_name = nil
         @use_json = true
     end
     
+    def use_checkpoint(filename)
+        @checkpoint_name = filename
+        
+        return self
+    end
+    
     def parse_events
         @use_json = false
         
@@ -67,7 +74,10 @@ class GHAProvider
     
     def logger=(logger)
         @logger = logger
+        
+        return self
     end
+    alias :use_logger :logger=
     
     def get(date)
         raise "Not implemented"
@@ -91,11 +101,45 @@ class GHAProvider
         return self
     end
     
+    def restore_checkpoint(from)
+        if @checkpoint_name && FileTest.exist?(@checkpoint_name)
+            # Note that this throws an exception if the file is not readable. This is the intended behavior.
+            # As opposed to that, failing to save the checkpoint information just results in a warning on the log.
+            loaded_from = Marshal.load(File.read(@checkpoint_name))
+            raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
+            
+            @logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
+            
+            return loaded_from
+        else
+            return from
+        end
+    end
+    
+    def update_checkpoint(current_time)
+        if @checkpoint_name
+            begin
+                File.open(@checkpoint_name, "wb") do |f|
+                    f.write(Marshal.dump(current_time))
+                end
+            rescue
+                @logger.warn(
+                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                )
+            end
+        end
+    end
+    
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         exceptions = []
         
+        from = restore_checkpoint(from)
+        
         self.each_time(from, to) do |current_time|
             events = []
+            
+            update_checkpoint(current_time)
+            
             begin
                 events = self.get(current_time)
             rescue GHAException => e
@@ -131,6 +175,8 @@ class GHAProvider
             GC.start
         end
         
+        update_checkpoint(to)
+        
         return exceptions
     end
     
@@ -142,13 +188,25 @@ class OnlineGHAProvider < GHAProvider
     def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
-        @max_retries = max_retries
-        @proactive = proactive
-        @proactive_pool_size = proactive_pool_size
-        @pool = Thread.pool(proactive_pool_size)
+        self.max_retries(max_retries)
+        self.proactive(proactive_pool_size) if proactive
+        
         @cache = Cache.new
     end
     
+    def max_retries(n)
+        @max_retries = n
+        
+        return self
+    end
+    
+    def proactive(pool_size = 10)
+        @proactive = true
+        @pool = Thread.pool(pool_size)
+        
+        return self
+    end
+    
     def get(current_time)        
         @max_retries.times do
             begin
@@ -214,10 +272,11 @@ class OnlineGHAProvider < GHAProvider
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
+            real_from = restore_checkpoint(from)
             any_ready = Thread.promise
             
             @logger.info("Proactively scheduling download tasks...")
-            self.each_time(from, to) do |current_time|
+            self.each_time(real_from, to) do |current_time|
                 @pool.process(current_time) do |current_time|
                     cache(current_time)
                     any_ready << true

data/lib/gh-archive/events.rb

@@ -1,5 +1,5 @@
 require 'time'
-require 'gh-archive/entities'
+require_relative File.expand_path('../entities', __FILE__)
 
 module GHArchive
     class Event

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.11'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-13 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -63,7 +63,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.21
-signing_key:
+rubygems_version: 3.2.22
+signing_key: 
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []