gh-archive 0.5 → 0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/gh-archive.rb +140 -39
- data/lib/gh-archive/entities.rb +312 -0
- data/lib/gh-archive/events.rb +405 -0
- metadata +8 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 498649edc20ceb260d6468cd7e9ff46781e148205c17cc2c6dd65cd8c7258e8f
|
4
|
+
data.tar.gz: 6e14d05254a7dcddc3a5afdf1c5e749602fdffc508df70dd1a01ebc97642a78c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b3156cdeec56577f61961b53c5552670534aaba0de3391daba5a99afc23fd09deb79da47879e2fc326eae4e85bbb180c41ca170540f67aa428c73571e443fad
|
7
|
+
data.tar.gz: 8129e09a30fd7abe02066ff890dcbd4d48f6965156f92d92582880883a11e508f397358aa39265fc65e7152c2e8204623a417c8689d3096c34e70df411e092f2
|
data/lib/gh-archive.rb
CHANGED
@@ -7,6 +7,8 @@ require 'tmpdir'
|
|
7
7
|
require 'thread/pool'
|
8
8
|
require 'thread/promise'
|
9
9
|
|
10
|
+
require_relative File.expand_path('../gh-archive/events', __FILE__)
|
11
|
+
|
10
12
|
module GHAUtils
|
11
13
|
def get_gha_filename(date)
|
12
14
|
return ("%04d-%02d-%02d-%d.json.gz" % [date.year, date.month, date.day, date.hour])
|
@@ -14,14 +16,19 @@ module GHAUtils
|
|
14
16
|
|
15
17
|
def read_gha_file_content(gz)
|
16
18
|
gzip = Zlib::GzipReader.new(gz)
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
return content
|
19
|
+
return gzip.read
|
20
|
+
ensure
|
21
|
+
gzip.close if gzip
|
21
22
|
end
|
22
23
|
|
23
|
-
def read_gha_file(
|
24
|
-
|
24
|
+
def read_gha_file(file)
|
25
|
+
if file.path.end_with?(".json")
|
26
|
+
content = file.read
|
27
|
+
elsif file.path.end_with?(".gz") || file.path.start_with?("/tmp/open-uri")
|
28
|
+
content = read_gha_file_content(file)
|
29
|
+
else
|
30
|
+
raise "Invalid file extension for #{file.path}: expected `.json.gz` or `json`,"
|
31
|
+
end
|
25
32
|
|
26
33
|
result = []
|
27
34
|
content.lines.each do |line|
|
@@ -31,11 +38,11 @@ module GHAUtils
|
|
31
38
|
return result
|
32
39
|
end
|
33
40
|
|
34
|
-
def
|
35
|
-
|
36
|
-
while
|
37
|
-
yield
|
38
|
-
|
41
|
+
def each_time(from, to)
|
42
|
+
current_time = from
|
43
|
+
while current_time < to
|
44
|
+
yield current_time
|
45
|
+
current_time += 3600
|
39
46
|
end
|
40
47
|
end
|
41
48
|
end
|
@@ -48,11 +55,29 @@ class GHAProvider
|
|
48
55
|
|
49
56
|
@includes = {}
|
50
57
|
@excludes = {}
|
58
|
+
|
59
|
+
@checkpoint_name = nil
|
60
|
+
@use_json = true
|
61
|
+
end
|
62
|
+
|
63
|
+
def use_checkpoint(filename)
|
64
|
+
@checkpoint_name = filename
|
65
|
+
|
66
|
+
return self
|
67
|
+
end
|
68
|
+
|
69
|
+
def parse_events
|
70
|
+
@use_json = false
|
71
|
+
|
72
|
+
return self
|
51
73
|
end
|
52
74
|
|
53
75
|
def logger=(logger)
|
54
76
|
@logger = logger
|
77
|
+
|
78
|
+
return self
|
55
79
|
end
|
80
|
+
alias :use_logger :logger=
|
56
81
|
|
57
82
|
def get(date)
|
58
83
|
raise "Not implemented"
|
@@ -63,6 +88,8 @@ class GHAProvider
|
|
63
88
|
@includes[key.to_s] = [] unless @includes[key.to_s]
|
64
89
|
@includes[key.to_s] << value
|
65
90
|
end
|
91
|
+
|
92
|
+
return self
|
66
93
|
end
|
67
94
|
|
68
95
|
def exclude(**args)
|
@@ -70,19 +97,48 @@ class GHAProvider
|
|
70
97
|
@excludes[key.to_s] = [] unless @excludes[key.to_s]
|
71
98
|
@excludes[key.to_s] << value
|
72
99
|
end
|
100
|
+
|
101
|
+
return self
|
73
102
|
end
|
74
103
|
|
75
104
|
def each(from = Time.gm(2015, 1, 1), to = Time.now)
|
76
|
-
|
105
|
+
exceptions = []
|
106
|
+
|
107
|
+
if @checkpoint_name && FileTest.exist?(@checkpoint_name)
|
108
|
+
# Note that this throws an exception if the file is not readable. This is the intended behavior.
|
109
|
+
# As opposed to that, failing to save the checkpoint information just results in a warning on the log.
|
110
|
+
loaded_from = Marshal.load(File.read(@checkpoint_name))
|
111
|
+
raise "The loaded checkpoint (#{loaded_from}) occurs before the current from date (#{from})" if loaded_from < from
|
112
|
+
|
113
|
+
@logger.info("Valid checkpoint loaded. Restored execution from #{loaded_from}.")
|
114
|
+
from = loaded_from
|
115
|
+
end
|
116
|
+
|
117
|
+
self.each_time(from, to) do |current_time|
|
77
118
|
events = []
|
78
119
|
begin
|
79
|
-
events = self.get(
|
80
|
-
|
81
|
-
|
82
|
-
|
120
|
+
events = self.get(current_time)
|
121
|
+
rescue GHAException => e
|
122
|
+
@logger.warn(e.message)
|
123
|
+
next
|
124
|
+
rescue => e
|
125
|
+
@logger.error("An exception occurred for #{current_time}: #{e.message}")
|
126
|
+
exceptions << e
|
83
127
|
next
|
84
128
|
end
|
85
129
|
|
130
|
+
if @checkpoint_name
|
131
|
+
begin
|
132
|
+
File.open(@checkpoint_name, "wb") do |f|
|
133
|
+
f.write(Marshal.dump(current_time))
|
134
|
+
end
|
135
|
+
rescue
|
136
|
+
@logger.warn(
|
137
|
+
"Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
|
138
|
+
)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
86
142
|
events.each do |event|
|
87
143
|
skip = false
|
88
144
|
@includes.each do |key, value|
|
@@ -94,12 +150,35 @@ class GHAProvider
|
|
94
150
|
end
|
95
151
|
next if skip
|
96
152
|
|
97
|
-
|
153
|
+
if @use_json
|
154
|
+
yield event, current_time
|
155
|
+
else
|
156
|
+
yield GHArchive::Event.parse(event), current_time
|
157
|
+
end
|
98
158
|
end
|
99
159
|
|
160
|
+
@logger.info("Scanned #{current_time}")
|
161
|
+
|
100
162
|
events.clear
|
101
163
|
GC.start
|
102
164
|
end
|
165
|
+
|
166
|
+
if @checkpoint_name
|
167
|
+
begin
|
168
|
+
File.open(@checkpoint_name, "wb") do |f|
|
169
|
+
f.write(Marshal.dump(to))
|
170
|
+
end
|
171
|
+
rescue
|
172
|
+
@logger.warn(
|
173
|
+
"Unable to save the checkpoint at the specified location (#{File.expand_path(@checkpoint_name)})."
|
174
|
+
)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
return exceptions
|
179
|
+
end
|
180
|
+
|
181
|
+
class GHAException < Exception
|
103
182
|
end
|
104
183
|
end
|
105
184
|
|
@@ -132,16 +211,21 @@ class OnlineGHAProvider < GHAProvider
|
|
132
211
|
return self.read_gha_file(gz)
|
133
212
|
end
|
134
213
|
end
|
135
|
-
rescue Errno::ECONNRESET
|
214
|
+
rescue Errno::ECONNRESET => e
|
215
|
+
@logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
|
136
216
|
next
|
137
|
-
rescue
|
138
|
-
|
139
|
-
|
140
|
-
|
217
|
+
rescue OpenURI::HTTPError => e
|
218
|
+
code = e.io.status[0]
|
219
|
+
if code.start_with?("5")
|
220
|
+
@logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
|
221
|
+
next
|
222
|
+
else
|
223
|
+
raise e
|
224
|
+
end
|
141
225
|
end
|
142
226
|
end
|
143
227
|
|
144
|
-
raise DownloadArchiveException, "Exceeded maximum number of tentative downloads."
|
228
|
+
raise DownloadArchiveException, "Exceeded maximum number of tentative downloads for #{current_time}."
|
145
229
|
end
|
146
230
|
|
147
231
|
def cache(current_time)
|
@@ -157,12 +241,17 @@ class OnlineGHAProvider < GHAProvider
|
|
157
241
|
@cache.put(filename, content)
|
158
242
|
return
|
159
243
|
end
|
160
|
-
rescue Errno::ECONNRESET
|
244
|
+
rescue Errno::ECONNRESET => e
|
245
|
+
@logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
|
161
246
|
next
|
162
|
-
rescue
|
163
|
-
|
164
|
-
|
165
|
-
|
247
|
+
rescue OpenURI::HTTPError => e
|
248
|
+
code = e.io.status[0]
|
249
|
+
if code.start_with?("5")
|
250
|
+
@logger.warn("A server error temporary prevented the download of #{current_time}: " + e.message)
|
251
|
+
next
|
252
|
+
else
|
253
|
+
raise e
|
254
|
+
end
|
166
255
|
end
|
167
256
|
end
|
168
257
|
end
|
@@ -172,11 +261,11 @@ class OnlineGHAProvider < GHAProvider
|
|
172
261
|
any_ready = Thread.promise
|
173
262
|
|
174
263
|
@logger.info("Proactively scheduling download tasks...")
|
175
|
-
self.
|
176
|
-
@pool.process(
|
177
|
-
cache(
|
264
|
+
self.each_time(from, to) do |current_time|
|
265
|
+
@pool.process(current_time) do |current_time|
|
266
|
+
cache(current_time)
|
178
267
|
any_ready << true
|
179
|
-
@logger.info("Proactively cached #{
|
268
|
+
@logger.info("Proactively cached #{current_time}. Cache size: #{@cache.size}")
|
180
269
|
end
|
181
270
|
end
|
182
271
|
|
@@ -221,7 +310,7 @@ class OnlineGHAProvider < GHAProvider
|
|
221
310
|
end
|
222
311
|
end
|
223
312
|
|
224
|
-
class DownloadArchiveException <
|
313
|
+
class DownloadArchiveException < GHAProvider::GHAException
|
225
314
|
end
|
226
315
|
end
|
227
316
|
|
@@ -234,8 +323,20 @@ class FolderGHAProvider < GHAProvider
|
|
234
323
|
|
235
324
|
def get(current_time)
|
236
325
|
filename = self.get_gha_filename(current_time)
|
237
|
-
File.
|
238
|
-
|
326
|
+
complete_filename = File.join(@folder, filename)
|
327
|
+
mode = "rb"
|
328
|
+
|
329
|
+
unless FileTest.exist?(complete_filename)
|
330
|
+
complete_filename = complete_filename.sub(".gz", "")
|
331
|
+
mode = "r"
|
332
|
+
end
|
333
|
+
|
334
|
+
unless FileTest.exist?(complete_filename)
|
335
|
+
raise GHAException.new("Cannot find any file (neither `.json.gz` nor `.json`) for #{current_time}")
|
336
|
+
end
|
337
|
+
|
338
|
+
File.open(complete_filename, mode) do |file|
|
339
|
+
return self.read_gha_file(file)
|
239
340
|
end
|
240
341
|
end
|
241
342
|
end
|
@@ -264,17 +365,17 @@ class GHADownloader
|
|
264
365
|
|
265
366
|
def download(from = Time.gm(2015, 1, 1), to = Time.now)
|
266
367
|
archive = []
|
267
|
-
self.
|
268
|
-
filename = self.get_gha_filename(
|
368
|
+
self.each_time(from, to) do |current_time|
|
369
|
+
filename = self.get_gha_filename(current_time)
|
269
370
|
out_filename = filename.clone
|
270
371
|
out_filename.gsub!(".json.gz", ".json") if @decompress
|
271
372
|
|
272
373
|
target_file = File.join(@folder, out_filename)
|
273
374
|
if FileTest.exist?(target_file)
|
274
|
-
@logger.info("Skipping existing file for #{
|
375
|
+
@logger.info("Skipping existing file for #{current_time}")
|
275
376
|
next
|
276
377
|
else
|
277
|
-
@logger.info("Downloading file for #{
|
378
|
+
@logger.info("Downloading file for #{current_time}")
|
278
379
|
end
|
279
380
|
|
280
381
|
File.open(target_file, 'w') do |f|
|
@@ -0,0 +1,312 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module GHArchive
|
4
|
+
Repository = Struct.new(:id, :name, :url)
|
5
|
+
CommitAuthor = Struct.new(:email, :name)
|
6
|
+
|
7
|
+
class Entity
|
8
|
+
def initialize(payload)
|
9
|
+
@payload = payload
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class Commit < Entity
|
14
|
+
def sha
|
15
|
+
@payload['sha']
|
16
|
+
end
|
17
|
+
|
18
|
+
def author
|
19
|
+
CommitAuthor.new(
|
20
|
+
@payload['author']['email'],
|
21
|
+
@payload['author']['name']
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def message
|
26
|
+
@payload['message']
|
27
|
+
end
|
28
|
+
|
29
|
+
def distinct
|
30
|
+
@payload['distinct']
|
31
|
+
end
|
32
|
+
|
33
|
+
def url
|
34
|
+
@payload['url']
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class User < Entity
|
39
|
+
def id
|
40
|
+
@payload['id']
|
41
|
+
end
|
42
|
+
|
43
|
+
def url
|
44
|
+
@payload['url']
|
45
|
+
end
|
46
|
+
|
47
|
+
def type
|
48
|
+
@payload['type']
|
49
|
+
end
|
50
|
+
|
51
|
+
def login
|
52
|
+
@payload['login']
|
53
|
+
end
|
54
|
+
|
55
|
+
def gravatar_id
|
56
|
+
@payload['gravatar_id']
|
57
|
+
end
|
58
|
+
|
59
|
+
def avatar_url
|
60
|
+
@payload['avatar_url']
|
61
|
+
end
|
62
|
+
|
63
|
+
def site_admin
|
64
|
+
@payload['site_admin']
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class BasicIssue < Entity
|
69
|
+
def url
|
70
|
+
@payload['url']
|
71
|
+
end
|
72
|
+
|
73
|
+
def id
|
74
|
+
@payload['id']
|
75
|
+
end
|
76
|
+
|
77
|
+
def number
|
78
|
+
@payload['number']
|
79
|
+
end
|
80
|
+
|
81
|
+
def state
|
82
|
+
@payload['state']
|
83
|
+
end
|
84
|
+
|
85
|
+
def locked
|
86
|
+
@payload['locked']
|
87
|
+
end
|
88
|
+
|
89
|
+
def title
|
90
|
+
@payload['title']
|
91
|
+
end
|
92
|
+
|
93
|
+
def body
|
94
|
+
@payload['body']
|
95
|
+
end
|
96
|
+
|
97
|
+
def user
|
98
|
+
User.new(@payload['user']) rescue nil
|
99
|
+
end
|
100
|
+
|
101
|
+
def created_at
|
102
|
+
Time.parse(@payload['created_at'])
|
103
|
+
end
|
104
|
+
|
105
|
+
def updated_at
|
106
|
+
Time.parse(@payload['updated_at']) rescue nil
|
107
|
+
end
|
108
|
+
|
109
|
+
def closed_at
|
110
|
+
Time.parse(@payload['closed_at']) rescue nil
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class PullRequest < BasicIssue
|
115
|
+
def merged_at
|
116
|
+
Time.parse(@payload['merged_at']) rescue nil
|
117
|
+
end
|
118
|
+
|
119
|
+
def merge_commit_sha
|
120
|
+
@payload['merge_commit_sha']
|
121
|
+
end
|
122
|
+
|
123
|
+
def merged
|
124
|
+
@payload['merged']
|
125
|
+
end
|
126
|
+
|
127
|
+
def mergeable
|
128
|
+
@payload['mergeable']
|
129
|
+
end
|
130
|
+
|
131
|
+
def mergeable_state
|
132
|
+
@payload['mergeable_state']
|
133
|
+
end
|
134
|
+
|
135
|
+
def merged_by
|
136
|
+
@payload['merged_by']
|
137
|
+
end
|
138
|
+
|
139
|
+
def comments
|
140
|
+
@payload['comments']
|
141
|
+
end
|
142
|
+
|
143
|
+
def review_comments
|
144
|
+
@payload['review_comments']
|
145
|
+
end
|
146
|
+
|
147
|
+
def commits
|
148
|
+
@payload['commits']
|
149
|
+
end
|
150
|
+
|
151
|
+
def additions
|
152
|
+
@payload['additions']
|
153
|
+
end
|
154
|
+
|
155
|
+
def deletions
|
156
|
+
@payload['deletions']
|
157
|
+
end
|
158
|
+
|
159
|
+
def changed_files
|
160
|
+
@payload['changed_files']
|
161
|
+
end
|
162
|
+
|
163
|
+
def head
|
164
|
+
@payload['head']
|
165
|
+
end
|
166
|
+
|
167
|
+
def base
|
168
|
+
@payload['base']
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
class Issue < BasicIssue
|
173
|
+
def labels
|
174
|
+
@payload['labels']
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
class BasicComment < Entity
|
179
|
+
def url
|
180
|
+
@payload['url']
|
181
|
+
end
|
182
|
+
|
183
|
+
def id
|
184
|
+
@payload['id']
|
185
|
+
end
|
186
|
+
|
187
|
+
def user
|
188
|
+
User.new(@payload['user']) rescue nil
|
189
|
+
end
|
190
|
+
|
191
|
+
def created_at
|
192
|
+
Time.parse(@payload['created_at'])
|
193
|
+
end
|
194
|
+
|
195
|
+
def updated_at
|
196
|
+
Time.parse(@payload['updated_at']) rescue nil
|
197
|
+
end
|
198
|
+
|
199
|
+
def body
|
200
|
+
@payload['body']
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
class PullRequestComment < BasicComment
|
205
|
+
def diff_hunk
|
206
|
+
@payload['diff_hunk']
|
207
|
+
end
|
208
|
+
|
209
|
+
def path
|
210
|
+
@payload['path']
|
211
|
+
end
|
212
|
+
|
213
|
+
def position
|
214
|
+
@payload['position']
|
215
|
+
end
|
216
|
+
|
217
|
+
def original_position
|
218
|
+
@payload['original_position']
|
219
|
+
end
|
220
|
+
|
221
|
+
def commit_id
|
222
|
+
@payload['commit_id']
|
223
|
+
end
|
224
|
+
|
225
|
+
def original_commit_id
|
226
|
+
@payload['original_commit_id']
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
class IssueComment < BasicComment
|
231
|
+
end
|
232
|
+
|
233
|
+
class Release < Entity
|
234
|
+
def url
|
235
|
+
@payload['url']
|
236
|
+
end
|
237
|
+
|
238
|
+
def id
|
239
|
+
@payload['id']
|
240
|
+
end
|
241
|
+
|
242
|
+
def tag_name
|
243
|
+
@payload['tag_name']
|
244
|
+
end
|
245
|
+
|
246
|
+
def target_commitish
|
247
|
+
@payload['target_commitish']
|
248
|
+
end
|
249
|
+
|
250
|
+
def name
|
251
|
+
@payload['name']
|
252
|
+
end
|
253
|
+
|
254
|
+
def draft
|
255
|
+
@payload['draft']
|
256
|
+
end
|
257
|
+
|
258
|
+
def author
|
259
|
+
User.new(@payload['author'])
|
260
|
+
end
|
261
|
+
|
262
|
+
def prerelease
|
263
|
+
@payload['prerelease']
|
264
|
+
end
|
265
|
+
|
266
|
+
def created_at
|
267
|
+
Time.parse(@payload['created_at'])
|
268
|
+
end
|
269
|
+
|
270
|
+
def published_at
|
271
|
+
Time.parse(@payload['published_at'])
|
272
|
+
end
|
273
|
+
|
274
|
+
def assets
|
275
|
+
@payload['assets']
|
276
|
+
end
|
277
|
+
|
278
|
+
def tarball_url
|
279
|
+
@payload['tarball_url']
|
280
|
+
end
|
281
|
+
|
282
|
+
def zipball_url
|
283
|
+
@payload['zipball_url']
|
284
|
+
end
|
285
|
+
|
286
|
+
def body
|
287
|
+
@payload['body']
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
class Page < Entity
|
292
|
+
def name
|
293
|
+
@payload['page_name']
|
294
|
+
end
|
295
|
+
|
296
|
+
def title
|
297
|
+
@payload['title']
|
298
|
+
end
|
299
|
+
|
300
|
+
def summary
|
301
|
+
@payload['summary']
|
302
|
+
end
|
303
|
+
|
304
|
+
def action
|
305
|
+
@payload['action']
|
306
|
+
end
|
307
|
+
|
308
|
+
def sha
|
309
|
+
@payload['sha']
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
@@ -0,0 +1,405 @@
|
|
1
|
+
require 'time'
|
2
|
+
require_relative File.expand_path('../entities', __FILE__)
|
3
|
+
|
4
|
+
module GHArchive
|
5
|
+
class Event
|
6
|
+
def self.parse(json)
|
7
|
+
IMPLEMENTATIONS.each do |event_class|
|
8
|
+
return event_class.new(json) if event_class.fits?(json)
|
9
|
+
end
|
10
|
+
|
11
|
+
return Event.new(json)
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(json)
|
15
|
+
@json = json.freeze
|
16
|
+
@payload = json['payload']
|
17
|
+
end
|
18
|
+
|
19
|
+
def public?
|
20
|
+
@json['public']
|
21
|
+
end
|
22
|
+
|
23
|
+
def created_at
|
24
|
+
Time.parse(@json['created_at'])
|
25
|
+
end
|
26
|
+
alias :time :created_at
|
27
|
+
|
28
|
+
def actor
|
29
|
+
User.new(@json['actor'])
|
30
|
+
end
|
31
|
+
|
32
|
+
def repo
|
33
|
+
Repository.new(
|
34
|
+
@json['repo']['id'],
|
35
|
+
@json['repo']['name'],
|
36
|
+
@json['repo']['url']
|
37
|
+
)
|
38
|
+
end
|
39
|
+
|
40
|
+
def json
|
41
|
+
@json
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class PushEvent < Event
|
46
|
+
def self.fits?(json)
|
47
|
+
json['type'] == "PushEvent"
|
48
|
+
end
|
49
|
+
|
50
|
+
def push_id
|
51
|
+
@payload['push_id']
|
52
|
+
end
|
53
|
+
|
54
|
+
def size
|
55
|
+
@payload['size']
|
56
|
+
end
|
57
|
+
|
58
|
+
def distinct_size
|
59
|
+
@payload['distinct_size']
|
60
|
+
end
|
61
|
+
|
62
|
+
def head
|
63
|
+
@payload['head']
|
64
|
+
end
|
65
|
+
|
66
|
+
def before
|
67
|
+
@payload['before']
|
68
|
+
end
|
69
|
+
|
70
|
+
def commits
|
71
|
+
@payload['commits'].map { |c| Commit.new(c) }
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class CommitCommentEvent < Event
|
76
|
+
def self.fits?(json)
|
77
|
+
return json['type'] == "CommitCommentEvent"
|
78
|
+
end
|
79
|
+
|
80
|
+
def comment_id
|
81
|
+
@payload['comment']['id']
|
82
|
+
end
|
83
|
+
|
84
|
+
def comment_url
|
85
|
+
@payload['comment']['url']
|
86
|
+
end
|
87
|
+
|
88
|
+
def comment_user
|
89
|
+
User.new(@payload['comment']['author'])
|
90
|
+
end
|
91
|
+
|
92
|
+
def comment_position
|
93
|
+
@payload['comment']['position']
|
94
|
+
end
|
95
|
+
|
96
|
+
def comment_line
|
97
|
+
@payload['comment']['line']
|
98
|
+
end
|
99
|
+
|
100
|
+
def comment_path
|
101
|
+
@payload['comment']['path']
|
102
|
+
end
|
103
|
+
|
104
|
+
def comment_commit_id
|
105
|
+
@payload['comment']['commit_id']
|
106
|
+
end
|
107
|
+
|
108
|
+
def comment_body
|
109
|
+
@payload['comment']['body']
|
110
|
+
end
|
111
|
+
|
112
|
+
def comment_created_at
|
113
|
+
Time.parse(@payload['comment']['created_at'])
|
114
|
+
end
|
115
|
+
|
116
|
+
def comment_updated_at
|
117
|
+
Time.parse(@payload['comment']['updated_at'])
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
class PullRequestEvent < Event
|
122
|
+
def self.fits?(json)
|
123
|
+
return json['type'] == "PullRequestEvent"
|
124
|
+
end
|
125
|
+
|
126
|
+
def action
|
127
|
+
@payload['action']
|
128
|
+
end
|
129
|
+
|
130
|
+
def number
|
131
|
+
@payload['number']
|
132
|
+
end
|
133
|
+
|
134
|
+
def pull_request
|
135
|
+
PullRequest.new(@payload['pull_request'])
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
class PullRequestReviewCommentEvent < Event
|
140
|
+
def self.fits?(json)
|
141
|
+
return json['type'] == "PullRequestReviewCommentEvent"
|
142
|
+
end
|
143
|
+
|
144
|
+
def action
|
145
|
+
@payload['action']
|
146
|
+
end
|
147
|
+
|
148
|
+
def number
|
149
|
+
@payload['number']
|
150
|
+
end
|
151
|
+
|
152
|
+
def pull_request
|
153
|
+
PullRequest.new(@payload['pull_request'])
|
154
|
+
end
|
155
|
+
|
156
|
+
def comment
|
157
|
+
PullRequestComment.new(@payload['comment'])
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
class IssuesEvent < Event
|
162
|
+
def self.fits?(json)
|
163
|
+
return json['type'] == "IssuesEvent"
|
164
|
+
end
|
165
|
+
|
166
|
+
def action
|
167
|
+
@payload['action']
|
168
|
+
end
|
169
|
+
|
170
|
+
def issue
|
171
|
+
Issue.new(@payload['issue'])
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
class IssueCommentEvent < Event
|
176
|
+
def self.fits?(json)
|
177
|
+
return json['type'] == "IssueCommentEvent"
|
178
|
+
end
|
179
|
+
|
180
|
+
def action
|
181
|
+
@payload['action']
|
182
|
+
end
|
183
|
+
|
184
|
+
def issue
|
185
|
+
Issue.new(@payload['issue'])
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
class CreateEvent < Event
|
190
|
+
def self.fits?(json)
|
191
|
+
return json['type'] == "CreateEvent"
|
192
|
+
end
|
193
|
+
|
194
|
+
def ref
|
195
|
+
@payload['ref']
|
196
|
+
end
|
197
|
+
|
198
|
+
def ref_type
|
199
|
+
@payload['ref_type']
|
200
|
+
end
|
201
|
+
|
202
|
+
def master_branch
|
203
|
+
@payload['master_branch']
|
204
|
+
end
|
205
|
+
|
206
|
+
def description
|
207
|
+
@payload['description']
|
208
|
+
end
|
209
|
+
|
210
|
+
def pusher_type
|
211
|
+
@payload['pusher_type']
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
class ForkEvent < Event
|
216
|
+
def self.fits?(json)
|
217
|
+
return json['type'] == "ForkEvent"
|
218
|
+
end
|
219
|
+
|
220
|
+
def forkee_id
|
221
|
+
@payload['forkee']['id']
|
222
|
+
end
|
223
|
+
|
224
|
+
def forkee_name
|
225
|
+
@payload['forkee']['name']
|
226
|
+
end
|
227
|
+
|
228
|
+
def forkee_full_name
|
229
|
+
@payload['forkee']['full_name']
|
230
|
+
end
|
231
|
+
|
232
|
+
def forkee_owner
|
233
|
+
User.new(@payload['forkee']['owner'])
|
234
|
+
end
|
235
|
+
|
236
|
+
def forkee_private
|
237
|
+
@payload['forkee']['private']
|
238
|
+
end
|
239
|
+
|
240
|
+
def forkee_description
|
241
|
+
@payload['forkee']['description']
|
242
|
+
end
|
243
|
+
|
244
|
+
def forkee_fork
|
245
|
+
@payload['forkee']['fork']
|
246
|
+
end
|
247
|
+
|
248
|
+
def forkee_created_at
|
249
|
+
Time.parse(@payload['forkee']['created_at'])
|
250
|
+
end
|
251
|
+
|
252
|
+
def forkee_updated_at
|
253
|
+
Time.parse(@payload['forkee']['updated_at'])
|
254
|
+
end
|
255
|
+
|
256
|
+
def forkee_pushed_at
|
257
|
+
Time.parse(@payload['forkee']['pushed_at'])
|
258
|
+
end
|
259
|
+
|
260
|
+
def forkee_urls
|
261
|
+
{
|
262
|
+
'git' => @payload['forkee']['git_url'],
|
263
|
+
'ssh' => @payload['forkee']['ssh_url'],
|
264
|
+
'clone' => @payload['forkee']['clone_url'],
|
265
|
+
'svn' => @payload['forkee']['svn_url']
|
266
|
+
}
|
267
|
+
end
|
268
|
+
|
269
|
+
def forkee_homepage
|
270
|
+
Time.parse(@payload['forkee']['homepage'])
|
271
|
+
end
|
272
|
+
|
273
|
+
def forkee_size
|
274
|
+
Time.parse(@payload['forkee']['size'])
|
275
|
+
end
|
276
|
+
|
277
|
+
def forkee_stargazers_count
|
278
|
+
Time.parse(@payload['forkee']['stargazers_count'])
|
279
|
+
end
|
280
|
+
|
281
|
+
def forkee_watchers_count
|
282
|
+
Time.parse(@payload['forkee']['watchers_count'])
|
283
|
+
end
|
284
|
+
|
285
|
+
def forkee_language
|
286
|
+
Time.parse(@payload['forkee']['language'])
|
287
|
+
end
|
288
|
+
|
289
|
+
def forkee_has_issues
|
290
|
+
Time.parse(@payload['forkee']['has_issues'])
|
291
|
+
end
|
292
|
+
|
293
|
+
def forkee_has_downloads
|
294
|
+
Time.parse(@payload['forkee']['has_downloads'])
|
295
|
+
end
|
296
|
+
|
297
|
+
def forkee_has_wiki
|
298
|
+
Time.parse(@payload['forkee']['has_wiki'])
|
299
|
+
end
|
300
|
+
|
301
|
+
def forkee_has_pages
|
302
|
+
Time.parse(@payload['forkee']['has_pages'])
|
303
|
+
end
|
304
|
+
|
305
|
+
def forkee_forks_count
|
306
|
+
Time.parse(@payload['forkee']['forks_count'])
|
307
|
+
end
|
308
|
+
|
309
|
+
def forkee_mirror_url
|
310
|
+
Time.parse(@payload['forkee']['mirror_url'])
|
311
|
+
end
|
312
|
+
|
313
|
+
def forkee_open_issues_count
|
314
|
+
Time.parse(@payload['forkee']['open_issues_count'])
|
315
|
+
end
|
316
|
+
|
317
|
+
def forkee_watchers
|
318
|
+
Time.parse(@payload['forkee']['watchers'])
|
319
|
+
end
|
320
|
+
|
321
|
+
def forkee_default_branch
|
322
|
+
Time.parse(@payload['forkee']['default_branch'])
|
323
|
+
end
|
324
|
+
|
325
|
+
def forkee_public
|
326
|
+
Time.parse(@payload['forkee']['public'])
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
class PublicEvent < Event
|
331
|
+
def self.fits?(json)
|
332
|
+
return json['type'] == "PublicEvent"
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
class WatchEvent < Event
|
337
|
+
def self.fits?(json)
|
338
|
+
return json['type'] == "WatchEvent"
|
339
|
+
end
|
340
|
+
|
341
|
+
def action
|
342
|
+
@payload['action']
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
class DeleteEvent < Event
|
347
|
+
def self.fits?(json)
|
348
|
+
return json['type'] == "DeleteEvent"
|
349
|
+
end
|
350
|
+
|
351
|
+
def ref
|
352
|
+
@payload['ref']
|
353
|
+
end
|
354
|
+
|
355
|
+
def ref_type
|
356
|
+
@payload['ref_type']
|
357
|
+
end
|
358
|
+
|
359
|
+
def pusher_type
|
360
|
+
@payload['pusher_type']
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
class ReleaseEvent < Event
|
365
|
+
def self.fits?(json)
|
366
|
+
return json['type'] == "ReleaseEvent"
|
367
|
+
end
|
368
|
+
|
369
|
+
def action
|
370
|
+
@payload['action']
|
371
|
+
end
|
372
|
+
|
373
|
+
def release
|
374
|
+
Release.new(@payload['release'])
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
class MemberEvent < Event
|
379
|
+
def self.fits?(json)
|
380
|
+
return json['type'] == "MemberEvent"
|
381
|
+
end
|
382
|
+
|
383
|
+
def action
|
384
|
+
@payload['action']
|
385
|
+
end
|
386
|
+
|
387
|
+
def member
|
388
|
+
User.new(@payload['member'])
|
389
|
+
end
|
390
|
+
end
|
391
|
+
|
392
|
+
class GollumEvent < Event
|
393
|
+
def self.fits?(json)
|
394
|
+
return json['type'] == "GollumEvent"
|
395
|
+
end
|
396
|
+
|
397
|
+
def pages
|
398
|
+
@payload[pages].map { |p| Page.new(p) }
|
399
|
+
end
|
400
|
+
end
|
401
|
+
|
402
|
+
class Event
|
403
|
+
IMPLEMENTATIONS = ObjectSpace.each_object(Class).select { |klass| klass < self }
|
404
|
+
end
|
405
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gh-archive
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.9'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Simone Scalabrino
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: code-assertions
|
@@ -57,11 +57,13 @@ extensions: []
|
|
57
57
|
extra_rdoc_files: []
|
58
58
|
files:
|
59
59
|
- lib/gh-archive.rb
|
60
|
+
- lib/gh-archive/entities.rb
|
61
|
+
- lib/gh-archive/events.rb
|
60
62
|
homepage: https://github.com/intersimone999/gh-archive
|
61
63
|
licenses:
|
62
64
|
- GPL-3.0-only
|
63
65
|
metadata: {}
|
64
|
-
post_install_message:
|
66
|
+
post_install_message:
|
65
67
|
rdoc_options: []
|
66
68
|
require_paths:
|
67
69
|
- lib
|
@@ -76,8 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
76
78
|
- !ruby/object:Gem::Version
|
77
79
|
version: '0'
|
78
80
|
requirements: []
|
79
|
-
rubygems_version: 3.2.
|
80
|
-
signing_key:
|
81
|
+
rubygems_version: 3.2.22
|
82
|
+
signing_key:
|
81
83
|
specification_version: 4
|
82
84
|
summary: GitHub Archive mining utility
|
83
85
|
test_files: []
|