gh-archive 0.4 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a7b3d8b97242ec787b56b409e4a81823473adc2b2915d5466e38a29015564a9
-  data.tar.gz: 6c50ded9b466fedb0ea1ca22dfcf8bdf9ec65a83b4666a429122a7345d0ebdea
+  metadata.gz: 011777addb798b172d58ffaac2b509ecf85288ee90cd28726c6303d14d39db1b
+  data.tar.gz: d8714b155567039e5de81f5ae36473c291f0af86701afaebf4527ab962dca240
 SHA512:
-  metadata.gz: 79bdb3c1649ff4fc86d711ce309e11055eb68720b8bf1d87f7e31b1a3a1586949cdd8d623630d446f7ec04288a2b0db7c598ca8bcf90c8a0c6c36edf6884805b
-  data.tar.gz: b790981c3d3becd6e46cece1e4bc5fa4cc6b7a25ac5c0bdc9fbbf6ab3ea6cb188f0d5abf48036426fec82e2fd1543ba85928f32df3c6915d56d2300eb35a43ce
+  metadata.gz: db6a72c3e6e31490c0a3b574ee0edf8f8995434f7ba32b6eb93c4ff35b3a8b0bd3e35c85ea207cb000b399fa71b524578067dc923752d5951f156b0f0d21df23
+  data.tar.gz: a0bac6036c2147e0bd933209f458cb272f6a20d669f2616fc4aa2e6b0a257354e704bcf48f3d658241d81d10d09a94ae07c899223862177fbe5315b9719b4874

data/lib/gh-archive.rb

@@ -4,6 +4,8 @@ require 'open-uri'
 require 'zlib'
 require 'logger'
 require 'tmpdir'
+require 'thread/pool'
+require 'thread/promise'
 
 module GHAUtils
     def get_gha_filename(date)
@@ -108,6 +110,7 @@ class OnlineGHAProvider < GHAProvider
         @max_retries = max_retries
         @proactive = proactive
         @proactive_pool_size = proactive_pool_size
+        @pool = Thread.pool(proactive_pool_size)
         @cache = Cache.new
     end
     
@@ -116,14 +119,23 @@ class OnlineGHAProvider < GHAProvider
             begin
                 filename = self.get_gha_filename(current_time)
                 
-                if @cache.has?(filename)
-                    result = @cache.get(filename)
+                if @proactive
+                    @logger.info("Waiting for cache to have #{current_time}...") unless @cache.has?(filename)
+                    
+                    while !@cache.has?(filename)
+                        sleep 1
+                    end
+
+                    return @cache.get(filename)
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                        # Save to cache
                         return self.read_gha_file(gz)
                     end
                 end
+            rescue Errno::ECONNRESET
+                next
+            rescue Zlib::GzipFile::Error
+                raise $!
             rescue
                 @logger.warn($!)
             end
@@ -133,59 +145,56 @@ class OnlineGHAProvider < GHAProvider
     end
     
     def cache(current_time)
+        @logger.info("Full cache. Waiting for some free slot...") if @cache.full?
+        while @cache.full?
+            sleep 1
+        end
         @max_retries.times do
             begin
                 filename = self.get_gha_filename(current_time)
-                
                 URI.open("http://data.gharchive.org/#{filename}") do |gz|
                     content = self.read_gha_file(gz)
                     @cache.put(filename, content)
                     return
                 end
+            rescue Errno::ECONNRESET
+                next
+            rescue Zlib::GzipFile::Error
+                raise $!
             rescue
-                p $!
+                @logger.warn($!)
             end
         end
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
-            @logger.info("Proactive download thread started")
-            Thread.start do
-                pool = []
-                self.each_date(from, to) do |current_date|
-                    while pool.size > @proactive_pool_size || @cache.full?
-                        pool.delete_if { |t| !t.alive? }
-                        sleep 0.1
-                    end
-                    
-                    pool << Thread.start do
-                        self.cache(current_date)
-                        @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
-                    end
-                    
-                    pool.delete_if { |t| !t.alive? }
+            any_ready = Thread.promise
+            
+            @logger.info("Proactively scheduling download tasks...")
+            self.each_date(from, to) do |current_date|
+                @pool.process(current_date) do |current_date|
+                    cache(current_date)
+                    any_ready << true
+                    @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
                 end
             end
+            
+            ~any_ready
+            @logger.info("Download tasks successfully scheduled!")
         end
         
         super
     end
     
     class Cache
-        def initialize(folder = Dir.mktmpdir, max_size = 100)
+        def initialize(max_size = 10)
             @cache = {}
             @max_size = max_size
-            @folder = folder
             @mutex = Mutex.new
         end
         
         def put(name, content)
-            #filename = "#@folder/#{name}"
-            #File.open(filename, 'w') do |f|
-                #f << content
-            #end
-            
             @mutex.synchronize do
                 @cache[name] = content
             end
@@ -195,18 +204,6 @@ class OnlineGHAProvider < GHAProvider
             @mutex.synchronize do
                 return @cache.delete(name)
             end
-        ensure
-            #self.unload(name)
-        end
-        
-        def unload(name)
-            File.unlink(@cache[name])
-            
-            @mutex.synchronize do
-                @cache.delete(name)
-            end
-            
-            return true
         end
         
         def size

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
 platform: ruby
 authors:
 - Simone Scalabrino
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -30,6 +30,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 description: Download and analyze the GitHub events stored at GitHub archive
 email: s.scalabrino9@gmail.com
 executables: []