gh-archive 0.1 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c493a4b789b2fc8b8b544287614667417f668c080e086a98e142eb70e5ada40
-  data.tar.gz: ab043dd0e56f9fa884405de70e86d1e8ff7b0091cbd980cd4c7be8694fcc3d77
+  metadata.gz: 011777addb798b172d58ffaac2b509ecf85288ee90cd28726c6303d14d39db1b
+  data.tar.gz: d8714b155567039e5de81f5ae36473c291f0af86701afaebf4527ab962dca240
 SHA512:
-  metadata.gz: 663226f4cd9b6dd51d679848877f1b93f64069a839c35d2149511135a960a07736589e8aa69b01d60c9ebdc295db25fb3cde6fa24f1a11885958aba9ffac0af1
-  data.tar.gz: f8889d87fb7853ae54871ccf192527d6021535d12c614fc4059a235e1df9c6245516d5ac9478f53c613932ca4a0655dd2e5c4a29955b8cacd843b38f99ddcba2
+  metadata.gz: db6a72c3e6e31490c0a3b574ee0edf8f8995434f7ba32b6eb93c4ff35b3a8b0bd3e35c85ea207cb000b399fa71b524578067dc923752d5951f156b0f0d21df23
+  data.tar.gz: a0bac6036c2147e0bd933209f458cb272f6a20d669f2616fc4aa2e6b0a257354e704bcf48f3d658241d81d10d09a94ae07c899223862177fbe5315b9719b4874

data/lib/gh-archive.rb

@@ -3,6 +3,9 @@ require 'json'
 require 'open-uri'
 require 'zlib'
 require 'logger'
+require 'tmpdir'
+require 'thread/pool'
+require 'thread/promise'
 
 module GHAUtils
     def get_gha_filename(date)
@@ -91,7 +94,7 @@ class GHAProvider
                 end
                 next if skip
                 
-                yield event
+                yield event, current_date
             end
             
             events.clear
@@ -101,11 +104,13 @@ class GHAProvider
 end
 
 class OnlineGHAProvider < GHAProvider
-    def initialize(max_retries = 3, proactive = false)
+    def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
         @max_retries = max_retries
         @proactive = proactive
+        @proactive_pool_size = proactive_pool_size
+        @pool = Thread.pool(proactive_pool_size)
         @cache = Cache.new
     end
     
@@ -114,16 +119,25 @@ class OnlineGHAProvider < GHAProvider
             begin
                 filename = self.get_gha_filename(current_time)
                 
-                if @cache.has?(filename)
-                    result = self.read_gha_file(@cache.get(filename))
+                if @proactive
+                    @logger.info("Waiting for cache to have #{current_time}...") unless @cache.has?(filename)
+                    
+                    while !@cache.has?(filename)
+                        sleep 1
+                    end
+
+                    return @cache.get(filename)
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                        # Save to cache
                         return self.read_gha_file(gz)
                     end
                 end
+            rescue Errno::ECONNRESET
+                next
+            rescue Zlib::GzipFile::Error
+                raise $!
             rescue
-                @logger.warning($!)
+                @logger.warn($!)
             end
         end
         
@@ -131,80 +145,70 @@ class OnlineGHAProvider < GHAProvider
     end
     
     def cache(current_time)
+        @logger.info("Full cache. Waiting for some free slot...") if @cache.full?
+        while @cache.full?
+            sleep 1
+        end
         @max_retries.times do
             begin
                 filename = self.get_gha_filename(current_time)
-                
                 URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                    @cache.put(filename, gz.read)
+                    content = self.read_gha_file(gz)
+                    @cache.put(filename, content)
                     return
                 end
+            rescue Errno::ECONNRESET
+                next
+            rescue Zlib::GzipFile::Error
+                raise $!
             rescue
+                @logger.warn($!)
             end
         end
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
-            @logger.info("Proactive download thread started")
-            Thread.start do
-                self.each_date(from, to) do |current_date|
-                    self.cache(current_date)
+            any_ready = Thread.promise
+            
+            @logger.info("Proactively scheduling download tasks...")
+            self.each_date(from, to) do |current_date|
+                @pool.process(current_date) do |current_date|
+                    cache(current_date)
+                    any_ready << true
                     @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
-                    
-                    if @cache.full?
-                        @logger.info("Full cache. Waiting...")
-                    end
-                    
-                    while @cache.full?
-                        sleep 1
-                    end
                 end
             end
+            
+            ~any_ready
+            @logger.info("Download tasks successfully scheduled!")
         end
         
         super
     end
     
     class Cache
-        def initialize(folder = Dir.mktmpdir, max_size = 100)
+        def initialize(max_size = 10)
             @cache = {}
             @max_size = max_size
-            @folder = folder
             @mutex = Mutex.new
         end
         
         def put(name, content)
-            File.open("#@folder/#{name}", 'w') do |f|
-                f << content
-            end
-            
             @mutex.synchronize do
-                @cache[name] = value
+                @cache[name] = content
             end
         end
         
         def get(name)
             @mutex.synchronize do
-                return File.read(@cache[name])
-            end
-        ensure
-            self.unload(name)
-        end
-        
-        def unload(name)
-            File.unlink(@cache[name])
-            
-            @mutex.synchronize do
-                @cache.delete(name)
+                return @cache.delete(name)
             end
-            
-            return true
         end
         
         def size
             @mutex.synchronize do
-                @cache.size
+                return @cache.size
             end
         end
         
@@ -243,15 +247,23 @@ class GHADownloader
         @logger = Logger.new(STDERR)
         @decompress = decompress
         @folder = folder
+        @max = nil
+        
         Dir.mkdir(@folder) unless FileTest.exist?(@folder)
         raise "A file exist with the desired folder name #{folder}" unless FileTest.directory?(@folder)
     end
     
+    def max(max)
+        @max = max
+        return self
+    end
+    
     def logger=(logger)
         @logger = logger
     end
     
     def download(from = Time.gm(2015, 1, 1), to = Time.now)
+        archive = []
         self.each_date(from, to) do |current_date|
             filename = self.get_gha_filename(current_date)
             out_filename = filename.clone
@@ -274,6 +286,13 @@ class GHADownloader
                     end
                 end
             end
+            archive << target_file
+            
+            if @max && archive.size > @max
+                last = archive.shift
+                @logger.info("Removing local file #{last}")
+                File.unlink(last)
+            end
             
             yield filename if block_given?
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.5'
 platform: ruby
 authors:
 - Simone Scalabrino
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -30,6 +30,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 description: Download and analyze the GitHub events stored at GitHub archive
 email: s.scalabrino9@gmail.com
 executables: []
@@ -56,7 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.21
 signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility