gh-archive 0.6 → 0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91b0e957c5176b791d4f49e382680865405e7a6b2b29b349bcbb78b92d884e02
-  data.tar.gz: f8ddae3d80e80a24931d8632c9798c8f01520e1fe5ac8a85079c0d0e85eadcbc
+  metadata.gz: b1a8f0a5088aa3d598466d5ef3ba3472b5e9b0c2ad3327f291c24045c01bdfba
+  data.tar.gz: b3df61810f7634eb8e9a153a812fa2073078975ee0ca303bfb9fb87e58f08160
 SHA512:
-  metadata.gz: f7b24be932f58142b36887671b4265e25631345e7b81cc36b264be4a018fc0c4a88b853ae384dc8472876bf0996e904bf499007adc3091ddb511f28c828090fc
-  data.tar.gz: 5cbb83495b9bb397a41022cb1bf4bce0344c735d16f9f43fb181b4b109948ac7a75bd44c693338f02b1ff17eeeeb5b83a6add9deeeedbd379a8848614041a3f5
+  metadata.gz: f5462029f6ef8e32f632bfdb7552606cf47a924e019502288ee2d841e28e76a5899d944d912a41933834e08e7a813c88cc919a7e93dba42b1c3250fcd911ae6e
+  data.tar.gz: 1f180c6640e38423ac2cde87ee4181cf04db8bb0a5cc6f861fd39d5de0c5bc4d30df87524f594482c170d591c5cd063abfa539b5715a4850056ab5262acef965

data/lib/gh-archive.rb

@@ -7,6 +7,8 @@ require 'tmpdir'
 require 'thread/pool'
 require 'thread/promise'
 
+require_relative File.expand_path('../gh-archive/events', __FILE__)
+
 module GHAUtils
     def get_gha_filename(date)
         return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
@@ -53,11 +55,29 @@ class GHAProvider
         
         @includes = {}
         @excludes = {}
+        
+        @checkpoint_name = nil
+        @use_json = true
+    end
+    
+    def use_checkpoint(filename)
+        @checkpoint_name = filename
+        
+        return self
+    end
+    
+    def parse_events
+        @use_json = false
+        
+        return self
     end
     
     def logger=(logger)
         @logger = logger
+        
+        return self
     end
+    alias :use_logger :logger=
     
     def get(date)
         raise "Not implemented"
@@ -68,6 +88,8 @@ class GHAProvider
             @includes[key.to_s] = [] unless @includes[key.to_s]
             @includes[key.to_s] << value
         end
+        
+        return self
     end
     
     def exclude(**args)
@@ -75,11 +97,44 @@ class GHAProvider
             @excludes[key.to_s] = [] unless @excludes[key.to_s]
             @excludes[key.to_s] << value
         end
+        
+        return self
+    end
+    
+    def restore_checkpoint(from)
+        if @checkpoint_name && FileTest.exist?(@checkpoint_name)
+            # Note that this throws an exception if the file is not readable. This is the intended behavior.
+            # As opposed to that, failing to save the checkpoint information just results in a warning on the log.
+            loaded_from = Marshal.load(File.read(@checkpoint_name))
+            raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
+            
+            @logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
+            
+            return loaded_from
+        else
+            return from
+        end
+    end
+    
+    def update_checkpoint(current_time)
+        if @checkpoint_name
+            begin
+                File.open(@checkpoint_name, "wb") do |f|
+                    f.write(Marshal.dump(current_time))
+                end
+            rescue
+                @logger.warn(
+                    "Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
+                )
+            end
+        end
     end
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         exceptions = []
         
+        from = restore_checkpoint(from)
+        
         self.each_time(from, to) do |current_time|
             events = []
             begin
@@ -93,6 +148,8 @@ class GHAProvider
                 next
             end
             
+            update_checkpoint(current_time)
+            
             events.each do |event|
                 skip = false
                 @includes.each do |key, value|
@@ -104,7 +161,11 @@ class GHAProvider
                 end
                 next if skip
                 
-                yield event, current_time
+                if @use_json
+                    yield event, current_time
+                else
+                    yield GHArchive::Event.parse(event), current_time
+                end
             end
             
             @logger.info("Scanned #{current_time}")
@@ -113,6 +174,8 @@ class GHAProvider
             GC.start
         end
         
+        update_checkpoint(to)
+        
         return exceptions
     end
     
@@ -196,10 +259,11 @@ class OnlineGHAProvider < GHAProvider
     
     def each(from = Time.gm(2015, 1, 1), to = Time.now)
         if @proactive
+            real_from = restore_checkpoint(from)
             any_ready = Thread.promise
             
             @logger.info("Proactively scheduling download tasks...")
-            self.each_time(from, to) do |current_time|
+            self.each_time(real_from, to) do |current_time|
                 @pool.process(current_time) do |current_time|
                     cache(current_time)
                     any_ready << true

data/lib/gh-archive/entities.rb

@@ -0,0 +1,312 @@
+require 'time'
+
+module GHArchive
+    Repository = Struct.new(:id, :name, :url)
+    CommitAuthor = Struct.new(:email, :name)
+    
+    class Entity
+        def initialize(payload)
+            @payload = payload
+        end
+    end
+    
+    class Commit < Entity
+        def sha
+            @payload['sha']
+        end
+        
+        def author
+            CommitAuthor.new(
+                @payload['author']['email'],
+                @payload['author']['name']
+            )
+        end
+        
+        def message
+            @payload['message']
+        end
+        
+        def distinct
+            @payload['distinct']
+        end
+        
+        def url
+            @payload['url']
+        end
+    end
+    
+    class User < Entity
+        def id
+            @payload['id']
+        end
+        
+        def url
+            @payload['url']
+        end
+        
+        def type
+            @payload['type']
+        end
+        
+        def login
+            @payload['login']
+        end
+        
+        def gravatar_id
+            @payload['gravatar_id']
+        end
+        
+        def avatar_url
+            @payload['avatar_url']
+        end
+        
+        def site_admin
+            @payload['site_admin']
+        end
+    end
+    
+    class BasicIssue < Entity                
+        def url
+            @payload['url']
+        end
+        
+        def id
+            @payload['id']
+        end
+        
+        def number
+            @payload['number']
+        end
+        
+        def state
+            @payload['state']
+        end
+        
+        def locked
+            @payload['locked']
+        end
+        
+        def title
+            @payload['title']
+        end
+        
+        def body
+            @payload['body']
+        end
+        
+        def user
+            User.new(@payload['user']) rescue nil
+        end
+        
+        def created_at
+            Time.parse(@payload['created_at'])
+        end
+        
+        def updated_at
+            Time.parse(@payload['updated_at']) rescue nil
+        end
+        
+        def closed_at
+            Time.parse(@payload['closed_at']) rescue nil
+        end
+    end
+    
+    class PullRequest < BasicIssue        
+        def merged_at
+            Time.parse(@payload['merged_at']) rescue nil
+        end
+        
+        def merge_commit_sha
+            @payload['merge_commit_sha']
+        end
+                
+        def merged
+            @payload['merged']
+        end
+        
+        def mergeable
+            @payload['mergeable']
+        end
+        
+        def mergeable_state
+            @payload['mergeable_state']
+        end
+        
+        def merged_by
+            @payload['merged_by']
+        end
+        
+        def comments
+            @payload['comments']
+        end
+        
+        def review_comments
+            @payload['review_comments']
+        end
+        
+        def commits
+            @payload['commits']
+        end
+        
+        def additions
+            @payload['additions']
+        end
+        
+        def deletions
+            @payload['deletions']
+        end
+        
+        def changed_files
+            @payload['changed_files']
+        end
+        
+        def head
+            @payload['head']
+        end
+        
+        def base
+            @payload['base']
+        end
+    end
+    
+    class Issue < BasicIssue        
+        def labels
+            @payload['labels']
+        end
+    end
+    
+    class BasicComment < Entity        
+        def url
+            @payload['url']
+        end
+        
+        def id
+            @payload['id']
+        end
+        
+        def user
+            User.new(@payload['user']) rescue nil
+        end
+        
+        def created_at
+            Time.parse(@payload['created_at'])
+        end
+        
+        def updated_at
+            Time.parse(@payload['updated_at']) rescue nil
+        end
+        
+        def body
+            @payload['body']
+        end
+    end
+    
+    class PullRequestComment < BasicComment
+        def diff_hunk
+            @payload['diff_hunk']
+        end
+        
+        def path
+            @payload['path']
+        end
+        
+        def position
+            @payload['position']
+        end
+        
+        def original_position
+            @payload['original_position']
+        end
+        
+        def commit_id
+            @payload['commit_id']
+        end
+        
+        def original_commit_id
+            @payload['original_commit_id']
+        end
+    end
+    
+    class IssueComment < BasicComment
+    end
+    
+    class Release < Entity
+        def url
+            @payload['url']
+        end
+        
+        def id
+            @payload['id']
+        end
+        
+        def tag_name
+            @payload['tag_name']
+        end
+        
+        def target_commitish
+            @payload['target_commitish']
+        end
+        
+        def name
+            @payload['name']
+        end
+        
+        def draft
+            @payload['draft']
+        end
+        
+        def author
+            User.new(@payload['author'])
+        end
+        
+        def prerelease
+            @payload['prerelease']
+        end
+        
+        def created_at
+            Time.parse(@payload['created_at'])
+        end
+        
+        def published_at
+            Time.parse(@payload['published_at'])
+        end
+        
+        def assets
+            @payload['assets']
+        end
+        
+        def tarball_url
+            @payload['tarball_url']
+        end
+        
+        def zipball_url
+            @payload['zipball_url']
+        end
+        
+        def body
+            @payload['body']
+        end
+    end
+    
+    class Page < Entity        
+        def name
+            @payload['page_name']
+        end
+        
+        def title
+            @payload['title']
+        end
+        
+        def summary
+            @payload['summary']
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def sha
+            @payload['sha']
+        end
+    end
+end

data/lib/gh-archive/events.rb

@@ -0,0 +1,405 @@
+require 'time'
+require_relative File.expand_path('../entities', __FILE__)
+
+module GHArchive
+    class Event
+        def self.parse(json)
+            IMPLEMENTATIONS.each do |event_class|
+                return event_class.new(json) if event_class.fits?(json)
+            end
+            
+            return Event.new(json)
+        end
+        
+        def initialize(json)
+            @json = json.freeze
+            @payload = json['payload']
+        end
+        
+        def public?
+            @json['public']
+        end
+        
+        def created_at
+            Time.parse(@json['created_at'])
+        end
+        alias :time :created_at
+        
+        def actor
+            User.new(@json['actor'])
+        end
+        
+        def repo
+            Repository.new(
+                @json['repo']['id'],
+                @json['repo']['name'],
+                @json['repo']['url']
+            )
+        end
+        
+        def json
+            @json
+        end
+    end
+    
+    class PushEvent < Event        
+        def self.fits?(json)
+            json['type'] == "PushEvent"
+        end
+        
+        def push_id
+            @payload['push_id']
+        end
+        
+        def size
+            @payload['size']
+        end
+        
+        def distinct_size
+            @payload['distinct_size']
+        end
+        
+        def head
+            @payload['head']
+        end
+        
+        def before
+            @payload['before']
+        end
+        
+        def commits
+            @payload['commits'].map { |c| Commit.new(c) }
+        end
+    end
+    
+    class CommitCommentEvent < Event
+        def self.fits?(json)
+            return json['type'] == "CommitCommentEvent"
+        end
+        
+        def comment_id
+            @payload['comment']['id']
+        end
+        
+        def comment_url
+            @payload['comment']['url']
+        end
+        
+        def comment_user
+            User.new(@payload['comment']['author'])
+        end
+        
+        def comment_position
+            @payload['comment']['position']
+        end
+        
+        def comment_line
+            @payload['comment']['line']
+        end
+        
+        def comment_path
+            @payload['comment']['path']
+        end
+        
+        def comment_commit_id
+            @payload['comment']['commit_id']
+        end
+        
+        def comment_body
+            @payload['comment']['body']
+        end
+        
+        def comment_created_at
+            Time.parse(@payload['comment']['created_at'])
+        end
+        
+        def comment_updated_at
+            Time.parse(@payload['comment']['updated_at'])
+        end
+    end
+    
+    class PullRequestEvent < Event
+        def self.fits?(json)
+            return json['type'] == "PullRequestEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def number
+            @payload['number']
+        end
+        
+        def pull_request
+            PullRequest.new(@payload['pull_request'])
+        end
+    end
+    
+    class PullRequestReviewCommentEvent < Event
+        def self.fits?(json)
+            return json['type'] == "PullRequestReviewCommentEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def number
+            @payload['number']
+        end
+        
+        def pull_request
+            PullRequest.new(@payload['pull_request'])
+        end
+        
+        def comment
+            PullRequestComment.new(@payload['comment'])
+        end
+    end
+    
+    class IssuesEvent < Event
+        def self.fits?(json)
+            return json['type'] == "IssuesEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def issue
+            Issue.new(@payload['issue'])
+        end
+    end
+    
+    class IssueCommentEvent < Event
+        def self.fits?(json)
+            return json['type'] == "IssueCommentEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def issue
+            Issue.new(@payload['issue'])
+        end
+    end
+    
+    class CreateEvent < Event
+        def self.fits?(json)
+            return json['type'] == "CreateEvent"
+        end
+        
+        def ref
+            @payload['ref']
+        end
+        
+        def ref_type
+            @payload['ref_type']
+        end
+        
+        def master_branch
+            @payload['master_branch']
+        end
+        
+        def description
+            @payload['description']
+        end
+        
+        def pusher_type
+            @payload['pusher_type']
+        end
+    end
+    
+    class ForkEvent < Event
+        def self.fits?(json)
+            return json['type'] == "ForkEvent"
+        end
+        
+        def forkee_id
+            @payload['forkee']['id']
+        end
+        
+        def forkee_name
+            @payload['forkee']['name']
+        end
+        
+        def forkee_full_name
+            @payload['forkee']['full_name']
+        end
+        
+        def forkee_owner
+            User.new(@payload['forkee']['owner'])
+        end
+        
+        def forkee_private
+            @payload['forkee']['private']
+        end
+        
+        def forkee_description
+            @payload['forkee']['description']
+        end
+        
+        def forkee_fork
+            @payload['forkee']['fork']
+        end
+        
+        def forkee_created_at
+            Time.parse(@payload['forkee']['created_at'])
+        end
+        
+        def forkee_updated_at
+            Time.parse(@payload['forkee']['updated_at'])
+        end
+        
+        def forkee_pushed_at
+            Time.parse(@payload['forkee']['pushed_at'])
+        end
+        
+        def forkee_urls
+            {
+                'git' => @payload['forkee']['git_url'],
+                'ssh' => @payload['forkee']['ssh_url'],
+                'clone' => @payload['forkee']['clone_url'],
+                'svn' => @payload['forkee']['svn_url']
+            }
+        end
+        
+        def forkee_homepage
+            Time.parse(@payload['forkee']['homepage'])
+        end
+        
+        def forkee_size
+            Time.parse(@payload['forkee']['size'])
+        end
+        
+        def forkee_stargazers_count
+            Time.parse(@payload['forkee']['stargazers_count'])
+        end
+        
+        def forkee_watchers_count
+            Time.parse(@payload['forkee']['watchers_count'])
+        end
+        
+        def forkee_language
+            Time.parse(@payload['forkee']['language'])
+        end
+        
+        def forkee_has_issues
+            Time.parse(@payload['forkee']['has_issues'])
+        end
+        
+        def forkee_has_downloads
+            Time.parse(@payload['forkee']['has_downloads'])
+        end
+        
+        def forkee_has_wiki
+            Time.parse(@payload['forkee']['has_wiki'])
+        end
+        
+        def forkee_has_pages
+            Time.parse(@payload['forkee']['has_pages'])
+        end
+        
+        def forkee_forks_count
+            Time.parse(@payload['forkee']['forks_count'])
+        end
+        
+        def forkee_mirror_url
+            Time.parse(@payload['forkee']['mirror_url'])
+        end
+        
+        def forkee_open_issues_count
+            Time.parse(@payload['forkee']['open_issues_count'])
+        end
+        
+        def forkee_watchers
+            Time.parse(@payload['forkee']['watchers'])
+        end
+        
+        def forkee_default_branch
+            Time.parse(@payload['forkee']['default_branch'])
+        end
+        
+        def forkee_public
+            Time.parse(@payload['forkee']['public'])
+        end
+    end
+    
+    class PublicEvent < Event
+        def self.fits?(json)
+            return json['type'] == "PublicEvent"
+        end
+    end
+    
+    class WatchEvent < Event
+        def self.fits?(json)
+            return json['type'] == "WatchEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+    end
+    
+    class DeleteEvent < Event
+        def self.fits?(json)
+            return json['type'] == "DeleteEvent"
+        end
+        
+        def ref
+            @payload['ref']
+        end
+        
+        def ref_type
+            @payload['ref_type']
+        end
+                
+        def pusher_type
+            @payload['pusher_type']
+        end
+    end
+    
+    class ReleaseEvent < Event
+        def self.fits?(json)
+            return json['type'] == "ReleaseEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def release
+            Release.new(@payload['release'])
+        end
+    end
+    
+    class MemberEvent < Event
+        def self.fits?(json)
+            return json['type'] == "MemberEvent"
+        end
+        
+        def action
+            @payload['action']
+        end
+        
+        def member
+            User.new(@payload['member'])
+        end
+    end
+    
+    class GollumEvent < Event
+        def self.fits?(json)
+            return json['type'] == "GollumEvent"
+        end
+        
+        def pages
+            @payload[pages].map { |p| Page.new(p) }
+        end
+    end
+    
+    class Event
+        IMPLEMENTATIONS = ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gh-archive
 version: !ruby/object:Gem::Version
-  version: '0.6'
+  version: '0.10'
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-13 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -57,11 +57,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/gh-archive.rb
+- lib/gh-archive/entities.rb
+- lib/gh-archive/events.rb
 homepage: https://github.com/intersimone999/gh-archive
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -76,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.21
-signing_key:
+rubygems_version: 3.2.22
+signing_key: 
 specification_version: 4
 summary: GitHub Archive mining utility
 test_files: []