gh-archive 0.8 → 0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4a9b89fb02620499e8b51f5c5e4ba34d0f3bc8b8f3ae4e2e69cba1e027bdb49
-  data.tar.gz: 6b707eb1bcb37b8a9b03ce36a2d4304e6760def0cc139ffad748a1596046f82f
+  metadata.gz: 498649edc20ceb260d6468cd7e9ff46781e148205c17cc2c6dd65cd8c7258e8f
+  data.tar.gz: 6e14d05254a7dcddc3a5afdf1c5e749602fdffc508df70dd1a01ebc97642a78c
 SHA512:
-  metadata.gz: '0975e354e028e768fb5bc4c17c19cddbc44b706394f60d8e22a64537a34342965bb98fa00365c329c30d189729dadba271ec6b101ff86593affef7e1d34848b3'
-  data.tar.gz: 95935fb3c27841760a68696c832e02108940f9eef535f5ed89afea9613a7988c622f2649f3feb77b22706250a30cfba30b5f36f2ada617f532e86a0426e052ce
+  metadata.gz: 6b3156cdeec56577f61961b53c5552670534aaba0de3391daba5a99afc23fd09deb79da47879e2fc326eae4e85bbb180c41ca170540f67aa428c73571e443fad
+  data.tar.gz: 8129e09a30fd7abe02066ff890dcbd4d48f6965156f92d92582880883a11e508f397358aa39265fc65e7152c2e8204623a417c8689d3096c34e70df411e092f2

data/lib/gh-archive.rb

@@ -56,9 +56,16 @@ class GHAProvider
         @includes = {}
         @excludes = {}
         
+        @checkpoint_name = nil
         @use_json = true
     end
     
+    def use_checkpoint(filename)
+        @checkpoint_name = filename
+        
+        return self
+    end
+    
     def parse_events
         @use_json = false
         
@@ -67,7 +74,10 @@ class GHAProvider
     
     def logger=(logger)
         @logger = logger
+        
+        return self
     end
+    alias :use_logger :logger=
     
     def get(date)
         raise "Not implemented"
@@ -94,6 +104,16 @@ class GHAProvider
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         exceptions = []
         
+        if @checkpoint_name && FileTest.exist?(@checkpoint_name)
+            # Note that this throws an exception if the file is not readable. This is the intended behavior.
+            # As opposed to that, failing to save the checkpoint information just results in a warning on the log.
+            loaded_from = Marshal.load(File.read(@checkpoint_name))
+            raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
+            
+            @logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
+            from = loaded_from
+        end
+        
         self.each_time(from, to) do |current_time|
             events = []
             begin
@@ -107,6 +127,18 @@ class GHAProvider
                 next
             end
             
+            if @checkpoint_name
+                begin
+                    File.open(@checkpoint_name, "wb") do |f|
+                        f.write(Marshal.dump(current_time))
+                    end
+                rescue
+                    @logger.warn(
+                        "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                    )
+                end
+            end
+            
             events.each do |event|
                 skip = false
                 @includes.each do |key, value|
@@ -131,6 +163,18 @@ class GHAProvider
             GC.start
         end
         
+        if @checkpoint_name
+            begin
+                File.open(@checkpoint_name, "wb") do |f|
+                    f.write(Marshal.dump(to))
+                end
+            rescue
+                @logger.warn(
+                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                )
+            end
+        end
+        
         return exceptions
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.8'
+  version: '0.9'
 platform: ruby
 authors:
 - Simone Scalabrino