gh-archive 0.2 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c90ec7c3f14e2f57de8a145ad50c1f7730869c5aad0b1de33286accec99642c7
-  data.tar.gz: a16c29db393499905695d411a62ca0dfb6e319d7fd509c104d41b2d0959b3414
+  metadata.gz: 91b0e957c5176b791d4f49e382680865405e7a6b2b29b349bcbb78b92d884e02
+  data.tar.gz: f8ddae3d80e80a24931d8632c9798c8f01520e1fe5ac8a85079c0d0e85eadcbc
 SHA512:
-  metadata.gz: f6a9feafaa7d0d06f75c489b147b2fc9b5485e88a28c15f98b1081c8b1f05fffc8218253ca817b0ecf64c1050f3e63ed4e1ad2d6aec9df5a4d362cf5ed2832bc
-  data.tar.gz: 1bfa5ab2dbccc74bd2b162066bab6874fc3a2fc08b6d2ad9ec02c05a73078ee84abe1d069eb2b51b491134b35ea4ee0724b81aa3ffbd845b9997ba3860639c05
+  metadata.gz: f7b24be932f58142b36887671b4265e25631345e7b81cc36b264be4a018fc0c4a88b853ae384dc8472876bf0996e904bf499007adc3091ddb511f28c828090fc
+  data.tar.gz: 5cbb83495b9bb397a41022cb1bf4bce0344c735d16f9f43fb181b4b109948ac7a75bd44c693338f02b1ff17eeeeb5b83a6add9deeeedbd379a8848614041a3f5

data/lib/gh-archive.rb

@@ -3,6 +3,9 @@ require 'json'
 require 'open-uri'
 require 'zlib'
 require 'logger'
+require 'tmpdir'
+require 'thread/pool'
+require 'thread/promise'
 
 module GHAUtils
     def get_gha_filename(date)
@@ -11,14 +14,19 @@ module GHAUtils
     
     def read_gha_file_content(gz)
         gzip = Zlib::GzipReader.new(gz)
-        content = gzip.read
-        gzip.close
-        
-        return content
+        return gzip.read
+    ensure
+        gzip.close if gzip
     end
     
-    def read_gha_file(gz)
-        content = read_gha_file_content(gz)
+    def read_gha_file(file)
+        if file.path.end_with?(".json")
+            content = file.read
+        elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
+            content = read_gha_file_content(file)
+        else
+            raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
+        end
             
         result = []
         content.lines.each do |line|
@@ -28,11 +36,11 @@ module GHAUtils
         return result
     end
     
-    def each_date(from, to)
-        current_date = from
-        while current_date < to
-            yield current_date
-            current_date += 3600
+    def each_time(from, to)
+        current_time = from
+        while current_time < to
+            yield current_time
+            current_time += 3600
         end
     end
 end
@@ -70,13 +78,18 @@ class GHAProvider
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
-        self.each_date(from, to) do |current_date|
+        exceptions = []
+        
+        self.each_time(from, to) do |current_time|
             events = []
             begin
-                events = self.get(current_date)
-                @logger.info("Scanned #{current_date}")
-            rescue
-                @logger.error($!)
+                events = self.get(current_time)
+            rescue GHAException => e
+                @logger.warn(e.message)
+                next
+            rescue => e
+                @logger.error("An exception occurred for #{current_time}: #{e.message}")
+                exceptions << e
                 next
             end
             
@@ -91,21 +104,30 @@ class GHAProvider
                 end
                 next if skip
                 
-                yield event
+                yield event, current_time
             end
             
+            @logger.info("Scanned #{current_time}")
+            
             events.clear
             GC.start
         end
+        
+        return exceptions
+    end
+    
+    class GHAException < Exception
     end
 end
 
 class OnlineGHAProvider < GHAProvider
-    def initialize(max_retries = 3, proactive = false)
+    def initialize(max_retries = 3, proactive = false, proactive_pool_size = 10)
         super()
         
         @max_retries = max_retries
         @proactive = proactive
+        @proactive_pool_size = proactive_pool_size
+        @pool = Thread.pool(proactive_pool_size)
         @cache = Cache.new
     end
     
@@ -114,97 +136,106 @@ class OnlineGHAProvider < GHAProvider
             begin
                 filename = self.get_gha_filename(current_time)
                 
-                if @cache.has?(filename)
-                    result = self.read_gha_file(@cache.get(filename))
+                if @proactive
+                    @logger.info("Waiting for cache to have #{current_time}...") unless @cache.has?(filename)
+                    
+                    while !@cache.has?(filename)
+                        sleep 1
+                    end
+
+                    return @cache.get(filename)
                 else
                     URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                        # Save to cache
                         return self.read_gha_file(gz)
                     end
                 end
-            rescue
-                @logger.warning($!)
+            rescue Errno::ECONNRESET => e
+                @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
+                next
+            rescue OpenURI::HTTPError => e
+                code = e.io.status[0]
+                if code.start_with?("5")
+                    @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
+                    next
+                else
+                    raise e
+                end
             end
         end
         
-        raise DownloadArchiveException, "Exceeded maximum number of tentative downloads."
+        raise DownloadArchiveException, "Exceeded maximum number of tentative downloads for #{current_time}."
     end
     
     def cache(current_time)
+        @logger.info("Full cache. Waiting for some free slot...") if @cache.full?
+        while @cache.full?
+            sleep 1
+        end
         @max_retries.times do
             begin
                 filename = self.get_gha_filename(current_time)
-                
                 URI.open("http://data.gharchive.org/#{filename}") do |gz|
-                    @cache.put(filename, gz.read)
+                    content = self.read_gha_file(gz)
+                    @cache.put(filename, content)
                     return
                 end
-            rescue
+            rescue Errno::ECONNRESET => e
+                @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
+                next
+            rescue OpenURI::HTTPError => e
+                code = e.io.status[0]
+                if code.start_with?("5")
+                    @logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
+                    next
+                else
+                    raise e
+                end
             end
         end
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
-            @logger.info("Proactive download thread started")
-            Thread.start do
-                self.each_date(from, to) do |current_date|
-                    self.cache(current_date)
-                    @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
-                    
-                    if @cache.full?
-                        @logger.info("Full cache. Waiting...")
-                    end
-                    
-                    while @cache.full?
-                        sleep 1
-                    end
+            any_ready = Thread.promise
+            
+            @logger.info("Proactively scheduling download tasks...")
+            self.each_time(from, to) do |current_time|
+                @pool.process(current_time) do |current_time|
+                    cache(current_time)
+                    any_ready << true
+                    @logger.info("Proactively cached #{current_time}. Cache size: #{@cache.size}")
                 end
             end
+            
+            ~any_ready
+            @logger.info("Download tasks successfully scheduled!")
         end
         
         super
     end
     
     class Cache
-        def initialize(folder = Dir.mktmpdir, max_size = 100)
+        def initialize(max_size = 10)
             @cache = {}
             @max_size = max_size
-            @folder = folder
             @mutex = Mutex.new
         end
         
         def put(name, content)
-            File.open("#@folder/#{name}", 'w') do |f|
-                f << content
-            end
-            
             @mutex.synchronize do
-                @cache[name] = value
+                @cache[name] = content
             end
         end
         
         def get(name)
             @mutex.synchronize do
-                return File.read(@cache[name])
-            end
-        ensure
-            self.unload(name)
-        end
-        
-        def unload(name)
-            File.unlink(@cache[name])
-            
-            @mutex.synchronize do
-                @cache.delete(name)
+                return @cache.delete(name)
             end
-            
-            return true
         end
         
         def size
             @mutex.synchronize do
-                @cache.size
+                return @cache.size
             end
         end
         
@@ -217,7 +248,7 @@ class OnlineGHAProvider < GHAProvider
         end
     end
     
-    class DownloadArchiveException < Exception
+    class DownloadArchiveException < GHAProvider::GHAException
     end
 end
 
@@ -230,8 +261,20 @@ class FolderGHAProvider < GHAProvider
     
     def get(current_time)        
         filename = self.get_gha_filename(current_time)
-        File.open(File.join(@folder, filename), "rb") do |gz|
-            return self.read_gha_file(gz)
+        complete_filename = File.join(@folder, filename)
+        mode = "rb"
+        
+        unless FileTest.exist?(complete_filename)
+            complete_filename = complete_filename.sub(".gz", "")
+            mode = "r"
+        end
+        
+        unless FileTest.exist?(complete_filename)
+            raise GHAException.new("Cannot find any file (neither `.json.gz` nor `.json`) for #{current_time}")
+        end
+        
+        File.open(complete_filename, mode) do |file|
+            return self.read_gha_file(file)
         end
     end
 end
@@ -260,17 +303,17 @@ class GHADownloader
     
     def download(from = Time.gm(2015, 1, 1), to = Time.now)
         archive = []
-        self.each_date(from, to) do |current_date|
-            filename = self.get_gha_filename(current_date)
+        self.each_time(from, to) do |current_time|
+            filename = self.get_gha_filename(current_time)
             out_filename = filename.clone
             out_filename.gsub!(".json.gz", ".json") if @decompress
             
             target_file = File.join(@folder, out_filename)
             if FileTest.exist?(target_file)
-                @logger.info("Skipping existing file for #{current_date}")
+                @logger.info("Skipping existing file for #{current_time}")
                 next
             else
-                @logger.info("Downloading file for #{current_date}")
+                @logger.info("Downloading file for #{current_time}")
             end
             
             File.open(target_file, 'w') do |f|

metadata

@@ -1,35 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.6'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.2
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 description: Download and analyze the GitHub events stored at GitHub archive
 email: s.scalabrino9@gmail.com
 executables: []
@@ -41,7 +61,7 @@ homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +76,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.21
+signing_key:
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []