gh-archive 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0c493a4b789b2fc8b8b544287614667417f668c080e086a98e142eb70e5ada40
+  data.tar.gz: ab043dd0e56f9fa884405de70e86d1e8ff7b0091cbd980cd4c7be8694fcc3d77
+SHA512:
+  metadata.gz: 663226f4cd9b6dd51d679848877f1b93f64069a839c35d2149511135a960a07736589e8aa69b01d60c9ebdc295db25fb3cde6fa24f1a11885958aba9ffac0af1
+  data.tar.gz: f8889d87fb7853ae54871ccf192527d6021535d12c614fc4059a235e1df9c6245516d5ac9478f53c613932ca4a0655dd2e5c4a29955b8cacd843b38f99ddcba2

data/lib/gh-archive.rb

@@ -0,0 +1,281 @@
+require 'code-assertions'
+require 'json'
+require 'open-uri'
+require 'zlib'
+require 'logger'
+
+module GHAUtils
+    def get_gha_filename(date)
+        return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
+    end
+    
+    def read_gha_file_content(gz)
+        gzip = Zlib::GzipReader.new(gz)
+        content = gzip.read
+        gzip.close
+        
+        return content
+    end
+    
+    def read_gha_file(gz)
+        content = read_gha_file_content(gz)
+            
+        result = []
+        content.lines.each do |line|
+            result << JSON.parse(line)
+        end
+        
+        return result
+    end
+    
+    def each_date(from, to)
+        current_date = from
+        while current_date < to
+            yield current_date
+            current_date += 3600
+        end
+    end
+end
+
+class GHAProvider
+    include GHAUtils
+    
+    def initialize
+        @logger = Logger.new(STDOUT)
+        
+        @includes = {}
+        @excludes = {}
+    end
+    
+    def logger=(logger)
+        @logger = logger
+    end
+    
+    def get(date)
+        raise "Not implemented"
+    end
+    
+    def include(**args)
+        args.each do |key, value|
+            @includes[key.to_s] = [] unless @includes[key.to_s]
+            @includes[key.to_s] << value
+        end
+    end
+    
+    def exclude(**args)
+        args.each do |key, value|
+            @excludes[key.to_s] = [] unless @excludes[key.to_s]
+            @excludes[key.to_s] << value
+        end
+    end
+    
+    def each(from = Time.gm(2015, 1, 1), to = Time.now)
+        self.each_date(from, to) do |current_date|
+            events = []
+            begin
+                events = self.get(current_date)
+                @logger.info("Scanned #{current_date}")
+            rescue
+                @logger.error($!)
+                next
+            end
+            
+            events.each do |event|
+                skip = false
+                @includes.each do |key, value|
+                    skip = true unless value.include?(event[key])
+                end
+                
+                @excludes.each do |key, value|
+                    skip = true if value.include?(event[key])
+                end
+                next if skip
+                
+                yield event
+            end
+            
+            events.clear
+            GC.start
+        end
+    end
+end
+
+class OnlineGHAProvider < GHAProvider
+    def initialize(max_retries = 3, proactive = false)
+        super()
+        
+        @max_retries = max_retries
+        @proactive = proactive
+        @cache = Cache.new
+    end
+    
+    def get(current_time)        
+        @max_retries.times do
+            begin
+                filename = self.get_gha_filename(current_time)
+                
+                if @cache.has?(filename)
+                    result = self.read_gha_file(@cache.get(filename))
+                else
+                    URI.open("http://data.gharchive.org/#{filename}") do |gz|
+                        # Save to cache
+                        return self.read_gha_file(gz)
+                    end
+                end
+            rescue
+                @logger.warning($!)
+            end
+        end
+        
+        raise DownloadArchiveException, "Exceeded maximum number of tentative downloads."
+    end
+    
+    def cache(current_time)
+        @max_retries.times do
+            begin
+                filename = self.get_gha_filename(current_time)
+                
+                URI.open("http://data.gharchive.org/#{filename}") do |gz|
+                    @cache.put(filename, gz.read)
+                    return
+                end
+            rescue
+            end
+        end
+    end
+    
+    def each(from = Time.gm(2015, 1, 1), to = Time.now)
+        if @proactive
+            @logger.info("Proactive download thread started")
+            Thread.start do
+                self.each_date(from, to) do |current_date|
+                    self.cache(current_date)
+                    @logger.info("Proactively cached #{current_date}. Cache size: #{@cache.size}")
+                    
+                    if @cache.full?
+                        @logger.info("Full cache. Waiting...")
+                    end
+                    
+                    while @cache.full?
+                        sleep 1
+                    end
+                end
+            end
+        end
+        
+        super
+    end
+    
+    class Cache
+        def initialize(folder = Dir.mktmpdir, max_size = 100)
+            @cache = {}
+            @max_size = max_size
+            @folder = folder
+            @mutex = Mutex.new
+        end
+        
+        def put(name, content)
+            File.open("#@folder/#{name}", 'w') do |f|
+                f << content
+            end
+            
+            @mutex.synchronize do
+                @cache[name] = value
+            end
+        end
+        
+        def get(name)
+            @mutex.synchronize do
+                return File.read(@cache[name])
+            end
+        ensure
+            self.unload(name)
+        end
+        
+        def unload(name)
+            File.unlink(@cache[name])
+            
+            @mutex.synchronize do
+                @cache.delete(name)
+            end
+            
+            return true
+        end
+        
+        def size
+            @mutex.synchronize do
+                @cache.size
+            end
+        end
+        
+        def has?(name)
+            return @cache.has_key?(name)
+        end
+        
+        def full?
+            self.size >= @max_size
+        end
+    end
+    
+    class DownloadArchiveException < Exception
+    end
+end
+
+class FolderGHAProvider < GHAProvider
+    def initialize(folder)
+        super()
+        
+        @folder = folder
+    end
+    
+    def get(current_time)        
+        filename = self.get_gha_filename(current_time)
+        File.open(File.join(@folder, filename), "rb") do |gz|
+            return self.read_gha_file(gz)
+        end
+    end
+end
+
+class GHADownloader
+    include GHAUtils
+    
+    def initialize(folder, decompress = false)
+        @logger = Logger.new(STDERR)
+        @decompress = decompress
+        @folder = folder
+        Dir.mkdir(@folder) unless FileTest.exist?(@folder)
+        raise "A file exist with the desired folder name #{folder}" unless FileTest.directory?(@folder)
+    end
+    
+    def logger=(logger)
+        @logger = logger
+    end
+    
+    def download(from = Time.gm(2015, 1, 1), to = Time.now)
+        self.each_date(from, to) do |current_date|
+            filename = self.get_gha_filename(current_date)
+            out_filename = filename.clone
+            out_filename.gsub!(".json.gz", ".json") if @decompress
+            
+            target_file = File.join(@folder, out_filename)
+            if FileTest.exist?(target_file)
+                @logger.info("Skipping existing file for #{current_date}")
+                next
+            else
+                @logger.info("Downloading file for #{current_date}")
+            end
+            
+            File.open(target_file, 'w') do |f|
+                URI.open("http://data.gharchive.org/#{filename}") do |gz|
+                    if @decompress
+                        f << self.read_gha_file_content(gz)
+                    else
+                        f << gz.read
+                    end
+                end
+            end
+            
+            yield filename if block_given?
+        end
+    end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: gh-archive
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Simone Scalabrino
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-01-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: code-assertions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+description: Download and analyze the GitHub events stored at GitHub archive
+email: s.scalabrino9@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/gh-archive.rb
+homepage: https://github.com/intersimone999/gh-archive
+licenses:
+- GPL-3.0-only
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: GitHub Archive mining utility
+test_files: []