gh-archive 0.11 → 0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 652cb53430d2e6a230d8f11465b81685f2e92107868345566d16b07d4c4231ed
-  data.tar.gz: abaec255c079e5a959fd51f7b1e7ca15d72c21eaaeac98d34d13309ec91eedeb
+  metadata.gz: de82207e7b0f5588a694e22b9d94c8e15e6812e8ad2dd433a8c78080c3629253
+  data.tar.gz: 7bc6e39cb4834922e29764744a3a753d8cea34052b44b282126b520b36f69d64
 SHA512:
-  metadata.gz: 829b1d53a7b72f4ea83712e8a95ba059e48dff7c982e811d30d27018e45e1b7010ace2a4a3f2641318a598e765a6d2c6138b319f9d67dd2604617dc2785175be
-  data.tar.gz: fd85da1c6630e8b534fb34fc9b5a08fcda55e3d2606ca393157134b6d85da2a9a7f92bc87ae31369441da1799cfd2178f0af9703ca997d10296c3f9a5ab1e4f9
+  metadata.gz: 1acd0b50fdba7cf5944e50670817ad41879ea273b27c79d87d6463b0bc438220b9a806aba2db879b4dbe8394439f24a66ec9bace2a466f51fbb44755eefd6d99
+  data.tar.gz: a30a6de0c073840eff54b7f0678bd996a794194d2a19ae35db1559b7d8ff4ec4f29591f36a6f114878d4737d29406ba919f8907b6706afb2f7b6ab7e8bcd4c26

data/lib/gh-archive.rb

@@ -9,6 +9,71 @@ require 'thread/promise'
 
 require_relative File.expand_path('../gh-archive/events', __FILE__)
 
+module GHArchive
+    class ThreadPool
+        def initialize(size)
+            @size = size
+            @threads = []
+            @queue = []
+            @mutex = Mutex.new
+            
+            @consumer_thread = Thread.start do
+                while !@shutdown || @threads.size > 0 || @queue.size > 0
+                    sleep 0.1 if @queue.size == 0 || @threads.size == @size
+                    @threads.delete_if { |t| !t.alive? }
+                    
+                    if @threads.size < @size && @queue.size > 0
+                        @mutex.synchronize do
+                            args, job = @queue.shift
+                            @threads << Thread.start(*args, &job)
+                        end
+                    end
+                end
+            end
+        end
+        
+        def process(*args, &block)
+            raise "Block expected" unless block_given?
+            raise "Can not add jobs while shutting down" if @shutdown
+            
+            @mutex.synchronize do
+                @queue << [args, block]
+            end
+            
+            return self.enqueued
+        end
+        
+        def shutdown
+            @shutdown = true
+        end
+        
+        def shutdown!
+            self.shutdown
+            @mutex.synchronize do
+                @queue.clear
+            end
+        end
+        
+        def enqueued
+            return @queue.size
+        end
+        
+        def shutdown?
+            @shutdown
+        end
+        
+        def alive?
+            @consumer_thread.alive?
+        end
+        
+        def wait
+            while alive?
+                sleep 0.1
+            end
+        end
+    end
+end
+
 module GHAUtils
     def get_gha_filename(date)
         return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
@@ -22,9 +87,10 @@ module GHAUtils
     end
     
     def read_gha_file(file)
-        if file.path.end_with?(".json")
+        
+        if !file.is_a?(StringIO) && file.path.end_with?(".json")
             content = file.read
-        elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
+        elsif file.is_a?(StringIO) || file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
             content = read_gha_file_content(file)
         else
             raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
@@ -202,7 +268,7 @@ class OnlineGHAProvider < GHAProvider
     
     def proactive(pool_size = 10)
         @proactive = true
-        @pool = Thread.pool(pool_size)
+        @pool = GHArchive::ThreadPool.new(pool_size)
         
         return self
     end
@@ -219,7 +285,12 @@ class OnlineGHAProvider < GHAProvider
                         sleep 1
                     end
 
-                    return @cache.get(filename)
+                    data = @cache.get(filename)
+                    if data
+                        return data
+                    else
+                        raise DownloadArchiveException, "Could not scan #{filename}: data unavailable."
+                    end
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
                         return self.read_gha_file(gz)
@@ -263,11 +334,17 @@ class OnlineGHAProvider < GHAProvider
                 if code.start_with?("5")
                     @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
                     next
+                elsif code == "404"
+                    @logger.error("File for #{current_time} not found. Skipping because: " + e.message)
                 else
                     raise e
                 end
+            rescue Zlib::GzipFile::Error => e
+                @logger.warn("Could not unzip, cache and analyze the zip at #{current_time}: " + e.message)
             end
         end
+        
+        @cache.put(filename, nil) unless @cache.has?(filename)
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.11'
+  version: '0.15'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -63,7 +63,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []