gh-archive 0.2 → 0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c90ec7c3f14e2f57de8a145ad50c1f7730869c5aad0b1de33286accec99642c7
-  data.tar.gz: a16c29db393499905695d411a62ca0dfb6e319d7fd509c104d41b2d0959b3414
+  metadata.gz: 0a12aa7954977bd9a8755f560530dd8db96b94e06f486e0a4155838ceb55e7f3
+  data.tar.gz: 8fc967a2e7a9bb0848c6e62a30231180c9a90ee96d72d6524c6b778ce54cbcbd
 SHA512:
-  metadata.gz: f6a9feafaa7d0d06f75c489b147b2fc9b5485e88a28c15f98b1081c8b1f05fffc8218253ca817b0ecf64c1050f3e63ed4e1ad2d6aec9df5a4d362cf5ed2832bc
-  data.tar.gz: 1bfa5ab2dbccc74bd2b162066bab6874fc3a2fc08b6d2ad9ec02c05a73078ee84abe1d069eb2b51b491134b35ea4ee0724b81aa3ffbd845b9997ba3860639c05
+  metadata.gz: 6f7c2ac9526de7e352825ed26c2ec3fc5375000e399529f45a07db7fbdaeef361f01a4782eb825f286e55311c3fc81dda7ecbcd509519e78da22147d1998771a
+  data.tar.gz: ab2b5f37f433ccde1d92406bd921c0c7a3a6ccaab41a3c11347a113c8c3db63beacf174ba6368be227302500b83bba59415f2cd85df59f9aff3f7f86f5c8b69c

data/lib/gh-archive.rb

@@ -3,6 +3,7 @@ require 'json'
 require 'open-uri'
 require 'zlib'
 require 'logger'
+require 'tmpdir'
 
 module GHAUtils
     def get_gha_filename(date)
@@ -101,11 +102,12 @@ class GHAProvider
 end
 
 class OnlineGHAProvider < GHAProvider
-    def initialize(max_retries = 3, proactive = false)
+    def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
         @max_retries = max_retries
         @proactive = proactive
+        @proactive_pool_size = proactive_pool_size
         @cache = Cache.new
     end
     
@@ -115,7 +117,7 @@ class OnlineGHAProvider < GHAProvider
                 filename = self.get_gha_filename(current_time)
                 
                 if @cache.has?(filename)
-                    result = self.read_gha_file(@cache.get(filename))
+                    result = @cache.get(filename)
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
                         # Save to cache
@@ -123,7 +125,7 @@ class OnlineGHAProvider < GHAProvider
                     end
                 end
             rescue
-                @logger.warning($!)
+                @logger.warn($!)
             end
         end
         
@@ -136,10 +138,12 @@ class OnlineGHAProvider < GHAProvider
                 filename = self.get_gha_filename(current_time)
                 
                 URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                    @cache.put(filename, gz.read)
+                    content = self.read_gha_file(gz)
+                    @cache.put(filename, content)
                     return
                 end
             rescue
+                p $!
             end
         end
     end
@@ -148,17 +152,19 @@ class OnlineGHAProvider < GHAProvider
         if @proactive
             @logger.info("Proactive download thread started")
             Thread.start do
+                pool = []
                 self.each_date(from, to) do |current_date|
-                    self.cache(current_date)
-                    @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
-                    
-                    if @cache.full?
-                        @logger.info("Full cache. Waiting...")
+                    while pool.size > @proactive_pool_size || @cache.full?
+                        pool.delete_if { |t| !t.alive? }
+                        sleep 0.1
                     end
                     
-                    while @cache.full?
-                        sleep 1
+                    pool << Thread.start do
+                        self.cache(current_date)
+                        @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
                     end
+                    
+                    pool.delete_if { |t| !t.alive? }
                 end
             end
         end
@@ -175,21 +181,22 @@ class OnlineGHAProvider < GHAProvider
         end
         
         def put(name, content)
-            File.open("#@folder/#{name}", 'w') do |f|
-                f << content
-            end
+            #filename = "#@folder/#{name}"
+            #File.open(filename, 'w') do |f|
+                #f << content
+            #end
             
             @mutex.synchronize do
-                @cache[name] = value
+                @cache[name] = content
             end
         end
         
         def get(name)
             @mutex.synchronize do
-                return File.read(@cache[name])
+                return @cache.delete(name)
             end
         ensure
-            self.unload(name)
+            #self.unload(name)
         end
         
         def unload(name)
@@ -204,7 +211,7 @@ class OnlineGHAProvider < GHAProvider
         
         def size
             @mutex.synchronize do
-                @cache.size
+                return @cache.size
             end
         end
         

metadata

@@ -1,33 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.3'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.2
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.2
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
 description: Download and analyze the GitHub events stored at GitHub archive
@@ -41,7 +41,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +56,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []