gh-archive 0.12 → 0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e36482fd8eeb76b12db28c1e68ad836d77a5985af6aa0e570a025bcb664b1a0
-  data.tar.gz: a91e869a8e3f614e8280f03749bcd31ce0de3c7c4e9dc9dd3d777c1a3f1e0d3f
+  metadata.gz: 1f339fca5ebd3f7ee085fa257b47567993ab776efd3ba143e424a6bab2ca1712
+  data.tar.gz: bd3709e9067fbc5ba0a7b92f156c3fa6f87d26bc81b0a969f7020526f31b5264
 SHA512:
-  metadata.gz: 4423afb5e0538be2abbe4ac21aeba3919081f0c61943ad9801add90b85b6f2e9df907836c13e00c6b300b28766c65c5cb2e78bc040393ebd38c9618092f2957d
-  data.tar.gz: c0caf8f4e47419744f7694748ed608c2e7da8fc0094477ce96bad211cf0b398ef5d43550fdcfd25ba1bc820cafd470ebc4f7526c39c171ef991bfc5fd2399882
+  metadata.gz: 9d56ecf4dc4101cf162d02e49f62dc77592bc652b9933b7413cb40f56eb25c595a20eed8fbc42e1f96830048d16520299972094b0d29b14f400befc5c21e1672
+  data.tar.gz: 5a7e8c158271b1b540e76e1b68f93ece7cebed8d29057728c6ac93b536f3f3417b89fffddcbe57c80e75269cda9ff719fef84e5bb0558dd52fd8842f80e44f55

data/lib/gh-archive.rb

@@ -285,7 +285,12 @@ class OnlineGHAProvider < GHAProvider
                         sleep 1
                     end
 
-                    return @cache.get(filename)
+                    data = @cache.get(filename)
+                    if data
+                        return data
+                    else
+                        raise DownloadArchiveException, "Could not scan #{filename}: data unavailable."
+                    end
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
                         return self.read_gha_file(gz)
@@ -313,9 +318,10 @@ class OnlineGHAProvider < GHAProvider
         while @cache.full?
             sleep 1
         end
+        
+        filename = self.get_gha_filename(current_time)
         @max_retries.times do
             begin
-                filename = self.get_gha_filename(current_time)
                 URI.open("http://data.gharchive.org/#{filename}") do |gz|
                     content = self.read_gha_file(gz)
                     @cache.put(filename, content)
@@ -329,11 +335,17 @@ class OnlineGHAProvider < GHAProvider
                 if code.start_with?("5")
                     @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
                     next
+                elsif code == "404"
+                    @logger.error("File for #{current_time} not found. Skipping because: " + e.message)
                 else
                     raise e
                 end
+            rescue Zlib::GzipFile::Error => e
+                @logger.warn("Could not unzip, cache and analyze the zip at #{current_time}: " + e.message)
             end
         end
+        
+        @cache.put(filename, nil) unless @cache.has?(filename)
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
@@ -383,7 +395,9 @@ class OnlineGHAProvider < GHAProvider
         end
         
         def has?(name)
-            return @cache.has_key?(name)
+            @mutex.synchronize do
+                return @cache.has_key?(name)
+            end
         end
         
         def full?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.12'
+  version: '0.16'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -63,7 +63,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []