ossert 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/.rspec +2 -0
- data/.rubocop_todo.yml +44 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +16 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +21 -0
- data/README.md +199 -0
- data/Rakefile +12 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/config/classifiers.yml +153 -0
- data/config/descriptions.yml +45 -0
- data/config/sidekiq.rb +15 -0
- data/config/stats.yml +198 -0
- data/config/translations.yml +44 -0
- data/db/backups/.keep +0 -0
- data/db/migrate/001_create_projects.rb +22 -0
- data/db/migrate/002_create_exceptions.rb +14 -0
- data/db/migrate/003_add_meta_to_projects.rb +14 -0
- data/db/migrate/004_add_timestamps_to_projects.rb +12 -0
- data/db/migrate/005_create_classifiers.rb +19 -0
- data/lib/ossert/classifiers/decision_tree.rb +112 -0
- data/lib/ossert/classifiers/growing/check.rb +172 -0
- data/lib/ossert/classifiers/growing/classifier.rb +175 -0
- data/lib/ossert/classifiers/growing.rb +163 -0
- data/lib/ossert/classifiers.rb +14 -0
- data/lib/ossert/config.rb +24 -0
- data/lib/ossert/fetch/bestgems.rb +98 -0
- data/lib/ossert/fetch/github.rb +536 -0
- data/lib/ossert/fetch/rubygems.rb +80 -0
- data/lib/ossert/fetch.rb +142 -0
- data/lib/ossert/presenters/project.rb +202 -0
- data/lib/ossert/presenters/project_v2.rb +117 -0
- data/lib/ossert/presenters.rb +8 -0
- data/lib/ossert/project.rb +144 -0
- data/lib/ossert/quarters_store.rb +164 -0
- data/lib/ossert/rake_tasks.rb +6 -0
- data/lib/ossert/reference.rb +87 -0
- data/lib/ossert/repositories.rb +138 -0
- data/lib/ossert/saveable.rb +153 -0
- data/lib/ossert/stats/agility_quarter.rb +62 -0
- data/lib/ossert/stats/agility_total.rb +71 -0
- data/lib/ossert/stats/base.rb +113 -0
- data/lib/ossert/stats/community_quarter.rb +28 -0
- data/lib/ossert/stats/community_total.rb +24 -0
- data/lib/ossert/stats.rb +32 -0
- data/lib/ossert/tasks/database.rake +179 -0
- data/lib/ossert/tasks/ossert.rake +52 -0
- data/lib/ossert/version.rb +4 -0
- data/lib/ossert/workers/fetch.rb +21 -0
- data/lib/ossert/workers/fetch_bestgems_page.rb +32 -0
- data/lib/ossert/workers/refresh_fetch.rb +22 -0
- data/lib/ossert/workers/sync_rubygems.rb +0 -0
- data/lib/ossert/workers.rb +11 -0
- data/lib/ossert.rb +63 -0
- data/ossert.gemspec +47 -0
- metadata +396 -0
@@ -0,0 +1,536 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'octokit'
|
3
|
+
|
4
|
+
module Ossert
|
5
|
+
module Fetch
|
6
|
+
class GitHub
|
7
|
+
attr_reader :client, :project
|
8
|
+
|
9
|
+
extend Forwardable
|
10
|
+
def_delegators :project, :agility, :community, :meta
|
11
|
+
|
12
|
+
def initialize(project)
|
13
|
+
@client = ::Octokit::Client.new(access_token: ENV['GITHUB_TOKEN'])
|
14
|
+
client.default_media_type = 'application/vnd.github.v3.star+json'
|
15
|
+
client.auto_paginate = true
|
16
|
+
|
17
|
+
@project = project
|
18
|
+
raise ArgumentError unless (@repo_name = project.github_alias).present?
|
19
|
+
@owner = @repo_name.split('/')[0]
|
20
|
+
@requests_count = 0
|
21
|
+
end
|
22
|
+
|
23
|
+
# TODO: Add github search feature
|
24
|
+
# def find_repo(user)
|
25
|
+
# first_found = client.search_repos(project.name, language: :ruby, user: user)[:items].first
|
26
|
+
# first_found.try(:[], :full_name)
|
27
|
+
# end
|
28
|
+
|
29
|
+
def request(endpoint, *args)
|
30
|
+
first_response_data = client.paginate(url(endpoint, args.shift), *args) do |_, last_response|
|
31
|
+
last_response.data.each { |data| yield data }
|
32
|
+
end
|
33
|
+
first_response_data.each { |data| yield data }
|
34
|
+
end
|
35
|
+
|
36
|
+
def url(endpoint, repo_name)
|
37
|
+
path = case endpoint
|
38
|
+
when /issues_comments/
|
39
|
+
'issues/comments'
|
40
|
+
when /pulls_comments/
|
41
|
+
'pulls/comments'
|
42
|
+
else
|
43
|
+
endpoint
|
44
|
+
end
|
45
|
+
"#{Octokit::Repository.path repo_name}/#{path}"
|
46
|
+
end
|
47
|
+
|
48
|
+
def issues(&block)
|
49
|
+
request(:issues, @repo_name, state: :all, &block)
|
50
|
+
end
|
51
|
+
|
52
|
+
def issues_comments(&block)
|
53
|
+
request(:issues_comments, @repo_name, &block)
|
54
|
+
end
|
55
|
+
|
56
|
+
def pulls(&block)
|
57
|
+
# fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
|
58
|
+
# http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
|
59
|
+
# fetch comments and link with PR by "pull_request_url"
|
60
|
+
request(:pulls, @repo_name, state: :all, &block)
|
61
|
+
end
|
62
|
+
|
63
|
+
def pulls_comments(&block)
|
64
|
+
# fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
|
65
|
+
# http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
|
66
|
+
# fetch comments and link with PR by "pull_request_url"
|
67
|
+
request(:pulls_comments, @repo_name, &block)
|
68
|
+
end
|
69
|
+
|
70
|
+
def contributors(&block)
|
71
|
+
request(:contributors, @repo_name, anon: true, &block)
|
72
|
+
end
|
73
|
+
|
74
|
+
def stargazers(&block)
|
75
|
+
request(:stargazers, @repo_name, &block)
|
76
|
+
end
|
77
|
+
|
78
|
+
def watchers(&block)
|
79
|
+
request(:subscribers, @repo_name, &block)
|
80
|
+
end
|
81
|
+
|
82
|
+
def forkers(&block)
|
83
|
+
request(:forks, @repo_name, &block)
|
84
|
+
end
|
85
|
+
|
86
|
+
def branches(&block)
|
87
|
+
request(:branches, @repo_name, &block)
|
88
|
+
end
|
89
|
+
|
90
|
+
def tags(&block)
|
91
|
+
request(:tags, @repo_name, &block)
|
92
|
+
end
|
93
|
+
|
94
|
+
def commits(from, to, &block)
|
95
|
+
request(:commits, @repo_name, since: from, until: to, &block)
|
96
|
+
end
|
97
|
+
|
98
|
+
def last_year_commits
|
99
|
+
last_year_commits = []
|
100
|
+
retry_count = 3
|
101
|
+
while last_year_commits.blank? && retry_count.positive?
|
102
|
+
last_year_commits = client.commit_activity_stats(@repo_name)
|
103
|
+
if last_year_commits.blank?
|
104
|
+
sleep(15 * retry_count)
|
105
|
+
retry_count -= 1
|
106
|
+
end
|
107
|
+
end
|
108
|
+
last_year_commits
|
109
|
+
end
|
110
|
+
|
111
|
+
def top_contributors
|
112
|
+
client.contributors_stats(@repo_name, retry_timeout: 5, retry_wait: 5)
|
113
|
+
end
|
114
|
+
|
115
|
+
def commit(sha)
|
116
|
+
client.commit(@repo_name, sha)
|
117
|
+
end
|
118
|
+
|
119
|
+
def tag_info(sha)
|
120
|
+
client.tag(@repo_name, sha)
|
121
|
+
rescue Octokit::NotFound
|
122
|
+
false
|
123
|
+
end
|
124
|
+
|
125
|
+
def date_from_tag(sha)
|
126
|
+
tag_info = tag_info(sha)
|
127
|
+
return tag_info[:tagger][:date] if tag_info
|
128
|
+
value = commit(sha)[:commit][:committer][:date]
|
129
|
+
DateTime.new(*value.split('-').map(&:to_i)).to_i
|
130
|
+
end
|
131
|
+
|
132
|
+
def commits_since(date)
|
133
|
+
client.commits_since(@repo_name, date)
|
134
|
+
end
|
135
|
+
|
136
|
+
def latest_release
|
137
|
+
@latest_release ||= client.latest_release(@repo_name)
|
138
|
+
end
|
139
|
+
|
140
|
+
# Add class with processing types, e.g. top_contributors, commits and so on
|
141
|
+
|
142
|
+
def process_top_contributors
|
143
|
+
@top_contributors = (top_contributors || []).map { |contrib_data| contrib_data[:author][:login] }
|
144
|
+
@top_contributors.last(10).reverse.each do |login|
|
145
|
+
(meta[:top_10_contributors] ||= []) << "https://github.com/#{login}"
|
146
|
+
end
|
147
|
+
nil
|
148
|
+
end
|
149
|
+
|
150
|
+
def process_commits
|
151
|
+
last_year_commits.each do |week|
|
152
|
+
current_count = agility.total.last_year_commits.to_i
|
153
|
+
agility.total.last_year_commits = current_count + week['total']
|
154
|
+
|
155
|
+
current_quarter_count = agility.quarters[week['week']].commits.to_i
|
156
|
+
agility.quarters[week['week']].commits = current_quarter_count + week['total']
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def process_last_release_date
|
161
|
+
latest_release_date = 0
|
162
|
+
|
163
|
+
tags do |tag|
|
164
|
+
tag_date = date_from_tag(tag[:commit][:sha])
|
165
|
+
latest_release_date = [latest_release_date, tag_date].max
|
166
|
+
|
167
|
+
agility.total.releases_total_gh << tag[:name]
|
168
|
+
agility.quarters[tag_date].releases_total_gh << tag[:name]
|
169
|
+
end
|
170
|
+
|
171
|
+
return if latest_release_date.zero?
|
172
|
+
|
173
|
+
agility.total.last_release_date = latest_release_date # wrong: last_release_commit[:commit][:committer][:date]
|
174
|
+
agility.total.commits_count_since_last_release = commits_since(Time.at(latest_release_date).utc).length
|
175
|
+
end
|
176
|
+
|
177
|
+
def process_quarters_issues_and_prs_processing_days
|
178
|
+
issues do |issue|
|
179
|
+
next if issue.key? :pull_request
|
180
|
+
next unless issue[:state] == 'closed'
|
181
|
+
next unless issue[:closed_at].present?
|
182
|
+
days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
|
183
|
+
(agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
|
184
|
+
end
|
185
|
+
|
186
|
+
pulls do |pull|
|
187
|
+
next unless pull[:state] == 'closed'
|
188
|
+
next unless pull[:closed_at].present?
|
189
|
+
days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
|
190
|
+
(agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def process_actual_prs_and_issues
|
195
|
+
actual_prs = Set.new
|
196
|
+
actual_issues = Set.new
|
197
|
+
agility.quarters.each_sorted do |_quarter, data|
|
198
|
+
data.pr_actual = actual_prs.to_a
|
199
|
+
data.issues_actual = actual_issues.to_a
|
200
|
+
|
201
|
+
closed = Set.new(data.pr_closed + data.issues_closed)
|
202
|
+
actual_prs = Set.new(actual_prs + data.pr_open) - closed
|
203
|
+
actual_issues = Set.new(actual_issues + data.issues_open) - closed
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def issue2pull_url(html_url)
|
208
|
+
html_url.gsub(
|
209
|
+
%r{https://github.com/(#{@repo_name})/pull/(\d+)},
|
210
|
+
'https://api.github.com/repos/\2/pulls/\3'
|
211
|
+
)
|
212
|
+
end
|
213
|
+
|
214
|
+
def process_open_pull(pull)
|
215
|
+
agility.total.pr_open << pull[:url]
|
216
|
+
agility.quarters[pull[:created_at]].pr_open << pull[:url]
|
217
|
+
end
|
218
|
+
|
219
|
+
def process_closed_pull(pull)
|
220
|
+
agility.total.pr_closed << pull[:url]
|
221
|
+
agility.quarters[pull[:created_at]].pr_open << pull[:url]
|
222
|
+
agility.quarters[pull[:closed_at]].pr_closed << pull[:url] if pull[:closed_at]
|
223
|
+
agility.quarters[pull[:merged_at]].pr_merged << pull[:url] if pull[:merged_at]
|
224
|
+
|
225
|
+
return unless pull[:closed_at].present?
|
226
|
+
days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
|
227
|
+
@pulls_processed_in_days << days_to_close
|
228
|
+
(agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
|
229
|
+
end
|
230
|
+
|
231
|
+
def process_users_from_pull(pull)
|
232
|
+
community.total.users_creating_pr << pull[:user][:login]
|
233
|
+
community.quarters[pull[:created_at]].users_creating_pr << pull[:user][:login]
|
234
|
+
community.total.users_involved << pull[:user][:login]
|
235
|
+
community.quarters[pull[:created_at]].users_involved << pull[:user][:login]
|
236
|
+
end
|
237
|
+
|
238
|
+
def process_pulls
|
239
|
+
@pulls_processed_in_days = Set.new
|
240
|
+
|
241
|
+
retry_call do
|
242
|
+
pulls do |pull|
|
243
|
+
case pull[:state]
|
244
|
+
when 'open'
|
245
|
+
process_open_pull(pull)
|
246
|
+
when 'closed'
|
247
|
+
process_closed_pull(pull)
|
248
|
+
end
|
249
|
+
|
250
|
+
if pull[:user][:login] == @owner
|
251
|
+
agility.total.pr_owner << pull[:url]
|
252
|
+
else
|
253
|
+
agility.total.pr_non_owner << pull[:url]
|
254
|
+
end
|
255
|
+
|
256
|
+
agility.total.pr_total << pull[:url]
|
257
|
+
agility.quarters[pull[:created_at]].pr_total << pull[:url]
|
258
|
+
|
259
|
+
created_at = pull[:created_at].to_datetime.to_i
|
260
|
+
if agility.total.first_pr_date.zero? || created_at < agility.total.first_pr_date
|
261
|
+
agility.total.first_pr_date = created_at
|
262
|
+
end
|
263
|
+
|
264
|
+
if agility.total.last_pr_date.zero? || created_at > agility.total.last_pr_date
|
265
|
+
agility.total.last_pr_date = created_at
|
266
|
+
end
|
267
|
+
|
268
|
+
process_users_from_pull(pull)
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
values = @pulls_processed_in_days.to_a.sort
|
273
|
+
agility.total.pr_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
|
274
|
+
agility.total.pr_processed_in_median = if values.count.odd?
|
275
|
+
values[values.count / 2]
|
276
|
+
elsif values.count.zero?
|
277
|
+
0
|
278
|
+
else
|
279
|
+
((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
|
280
|
+
end
|
281
|
+
|
282
|
+
|
283
|
+
retry_call do
|
284
|
+
pulls_comments do |pull_comment|
|
285
|
+
login = pull_comment[:user].try(:[], :login).presence || generate_anonymous
|
286
|
+
if community.total.contributors.include? login
|
287
|
+
agility.total.pr_with_contrib_comments << pull_comment[:pull_request_url]
|
288
|
+
end
|
289
|
+
|
290
|
+
community.total.users_commenting_pr << login
|
291
|
+
community.quarters[pull_comment[:created_at]].users_commenting_pr << login
|
292
|
+
community.total.users_involved << login
|
293
|
+
community.quarters[pull_comment[:created_at]].users_involved << login
|
294
|
+
end
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
def process_open_issue(issue)
|
299
|
+
agility.total.issues_open << issue[:url]
|
300
|
+
agility.quarters[issue[:created_at]].issues_open << issue[:url]
|
301
|
+
end
|
302
|
+
|
303
|
+
def process_closed_issue(issue)
|
304
|
+
agility.total.issues_closed << issue[:url]
|
305
|
+
# if issue is closed for now, it also was opened somewhen
|
306
|
+
agility.quarters[issue[:created_at]].issues_open << issue[:url]
|
307
|
+
agility.quarters[issue[:closed_at]].issues_closed << issue[:url] if issue[:closed_at]
|
308
|
+
|
309
|
+
return unless issue[:closed_at].present?
|
310
|
+
days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
|
311
|
+
@issues_processed_in_days << days_to_close
|
312
|
+
(agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
|
313
|
+
end
|
314
|
+
|
315
|
+
def process_users_from_issue(issue)
|
316
|
+
community.total.users_creating_issues << issue[:user][:login]
|
317
|
+
community.quarters[issue[:created_at]].users_creating_issues << issue[:user][:login]
|
318
|
+
community.total.users_involved << issue[:user][:login]
|
319
|
+
community.quarters[issue[:created_at]].users_involved << issue[:user][:login]
|
320
|
+
end
|
321
|
+
|
322
|
+
def process_issues
|
323
|
+
@issues_processed_in_days = []
|
324
|
+
|
325
|
+
issues do |issue|
|
326
|
+
next if issue.key? :pull_request
|
327
|
+
case issue[:state]
|
328
|
+
when 'open'
|
329
|
+
process_open_issue(issue)
|
330
|
+
when 'closed'
|
331
|
+
process_closed_issue(issue)
|
332
|
+
end
|
333
|
+
|
334
|
+
if issue[:user][:login] == @owner
|
335
|
+
agility.total.issues_owner << issue[:url]
|
336
|
+
else
|
337
|
+
agility.total.issues_non_owner << issue[:url]
|
338
|
+
end
|
339
|
+
|
340
|
+
agility.total.issues_total << issue[:url]
|
341
|
+
agility.quarters[issue[:created_at]].issues_total << issue[:url]
|
342
|
+
|
343
|
+
created_at = issue[:created_at].to_datetime.to_i
|
344
|
+
if agility.total.first_issue_date.zero? || created_at < agility.total.first_issue_date
|
345
|
+
agility.total.first_issue_date = created_at
|
346
|
+
end
|
347
|
+
|
348
|
+
if agility.total.last_issue_date.zero? || created_at > agility.total.last_issue_date
|
349
|
+
agility.total.last_issue_date = created_at
|
350
|
+
end
|
351
|
+
|
352
|
+
process_users_from_issue(issue)
|
353
|
+
end
|
354
|
+
|
355
|
+
values = @issues_processed_in_days.to_a.sort
|
356
|
+
agility.total.issues_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
|
357
|
+
agility.total.issues_processed_in_median = if values.count.odd?
|
358
|
+
values[values.count / 2]
|
359
|
+
elsif values.count.zero?
|
360
|
+
0
|
361
|
+
else
|
362
|
+
((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
|
363
|
+
end
|
364
|
+
|
365
|
+
issues_comments do |issue_comment|
|
366
|
+
login = issue_comment[:user].try(:[], :login).presence || generate_anonymous
|
367
|
+
issue_url = /\A(.*)#issuecomment.*\z/.match(issue_comment[:html_url])[1]
|
368
|
+
if issue_url.include?('/pull/') # PR comments are stored as Issue comments. Sadness =(
|
369
|
+
if community.total.contributors.include? login
|
370
|
+
agility.total.pr_with_contrib_comments << issue2pull_url(issue_url)
|
371
|
+
end
|
372
|
+
|
373
|
+
community.total.users_commenting_pr << login
|
374
|
+
community.quarters[issue_comment[:created_at]].users_commenting_pr << login
|
375
|
+
community.total.users_involved << login
|
376
|
+
community.quarters[issue_comment[:created_at]].users_involved << login
|
377
|
+
next
|
378
|
+
end
|
379
|
+
|
380
|
+
if community.total.contributors.include? login
|
381
|
+
agility.total.issues_with_contrib_comments << issue_url
|
382
|
+
end
|
383
|
+
|
384
|
+
community.total.users_commenting_issues << login
|
385
|
+
community.quarters[issue_comment[:created_at]].users_commenting_issues << login
|
386
|
+
community.total.users_involved << login
|
387
|
+
community.quarters[issue_comment[:created_at]].users_involved << login
|
388
|
+
end
|
389
|
+
end
|
390
|
+
|
391
|
+
def process
|
392
|
+
contributors do |c|
|
393
|
+
login = c.try(:[], :login).presence || generate_anonymous
|
394
|
+
community.total.contributors << login
|
395
|
+
end
|
396
|
+
community.total.users_involved += community.total.contributors
|
397
|
+
community.total.users_involved.uniq!
|
398
|
+
|
399
|
+
# TODO: extract contributors and commits, quarter by quarter.
|
400
|
+
#
|
401
|
+
# => {:sha=>"d1a43d32e615b4a75117151b002266c560ce9061",
|
402
|
+
# :commit=>
|
403
|
+
# {:author=>
|
404
|
+
# {:name=>"Yves Senn",
|
405
|
+
# :email=>"yves.senn@gmail.com",
|
406
|
+
# :date=>"2015-09-22T08:25:14Z"},
|
407
|
+
# :committer=>
|
408
|
+
# {:name=>"Yves Senn",
|
409
|
+
# :email=>"yves.senn@gmail.com",
|
410
|
+
# :date=>"2015-09-22T08:25:14Z"},
|
411
|
+
# :message=>
|
412
|
+
# "Merge pull request #21678 from ronakjangir47/array_to_formatted_s_docs\n\nAdded Examples in docs for int
|
413
|
+
# ernal behavior of Array#to_formatted_s [ci skip]",
|
414
|
+
# :tree=>
|
415
|
+
# {:sha=>"204811aa155645b461467dbd2238ac41c0fe8a30",
|
416
|
+
# :url=>
|
417
|
+
# "https://api.github.com/repos/rails/rails/git/trees/204811aa155645b461467dbd2238ac41c0fe8a30"},
|
418
|
+
# :url=>
|
419
|
+
# "https://api.github.com/repos/rails/rails/git/commits/d1a43d32e615b4a75117151b002266c560ce9061",
|
420
|
+
# :comment_count=>0},
|
421
|
+
# :url=>
|
422
|
+
# "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061",
|
423
|
+
# :html_url=>
|
424
|
+
# "https://github.com/rails/rails/commit/d1a43d32e615b4a75117151b002266c560ce9061",
|
425
|
+
# :comments_url=>
|
426
|
+
# "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061/comments",
|
427
|
+
# :author=>
|
428
|
+
# {:login=>"senny",
|
429
|
+
# ...
|
430
|
+
# :type=>"User",
|
431
|
+
# :site_admin=>false},
|
432
|
+
# :committer=>
|
433
|
+
# {:login=>"senny",
|
434
|
+
# ...
|
435
|
+
# :type=>"User",
|
436
|
+
# :site_admin=>false},
|
437
|
+
# :parents=>
|
438
|
+
# [{:sha=>"2a7e8f54c66dbd65822f2a7135546a240426b631",
|
439
|
+
# :url=>
|
440
|
+
# "https://api.github.com/repos/rails/rails/commits/2a7e8f54c66dbd65822f2a7135546a240426b631",
|
441
|
+
# :html_url=>
|
442
|
+
# "https://github.com/rails/rails/commit/2a7e8f54c66dbd65822f2a7135546a240426b631"},
|
443
|
+
# {:sha=>"192d29f1c7ea16c506c09da2b854d1acdfbc8749",
|
444
|
+
# :url=>
|
445
|
+
# "https://api.github.com/repos/rails/rails/commits/192d29f1c7ea16c506c09da2b854d1acdfbc8749",
|
446
|
+
# :html_url=>
|
447
|
+
# "https://github.com/rails/rails/commit/192d29f1c7ea16c506c09da2b854d1acdfbc8749"}]}
|
448
|
+
|
449
|
+
# process collaborators and commits. year by year, more info then ^^^^^
|
450
|
+
# count = 0; collab = Set.new; fetcher.commits(14.months.ago.utc.iso8601, 13.month.ago.utc.iso8601) do |commit|
|
451
|
+
# count+=1
|
452
|
+
# collab << (commit[:author].try(:[],:login) || commit[:commit][:author][:name])
|
453
|
+
# end
|
454
|
+
|
455
|
+
process_issues
|
456
|
+
|
457
|
+
process_pulls
|
458
|
+
|
459
|
+
process_actual_prs_and_issues
|
460
|
+
|
461
|
+
process_last_release_date
|
462
|
+
|
463
|
+
process_commits
|
464
|
+
|
465
|
+
sleep(1)
|
466
|
+
|
467
|
+
process_top_contributors
|
468
|
+
|
469
|
+
sleep(1)
|
470
|
+
|
471
|
+
branches do |branch|
|
472
|
+
# stale and total
|
473
|
+
# by quarter ? date from commit -> [:commit][:committer][:date]
|
474
|
+
# 1. save dates by commit sha.
|
475
|
+
branch_updated_at = commit(branch[:commit][:sha])[:commit][:committer][:date]
|
476
|
+
stale_threshold = Time.now.beginning_of_quarter
|
477
|
+
|
478
|
+
# 2. date -> total by quarter
|
479
|
+
# date -> stale
|
480
|
+
agility.total.branches << branch[:name]
|
481
|
+
agility.total.stale_branches << branch[:name] if branch_updated_at < stale_threshold
|
482
|
+
agility.quarters[branch_updated_at].branches << branch[:name]
|
483
|
+
end
|
484
|
+
|
485
|
+
stargazers do |stargazer|
|
486
|
+
login = stargazer[:user][:login].presence || generate_anonymous
|
487
|
+
community.total.stargazers << login
|
488
|
+
community.total.users_involved << login
|
489
|
+
|
490
|
+
community.quarters[stargazer[:starred_at]].stargazers << login
|
491
|
+
community.quarters[stargazer[:starred_at]].users_involved << login
|
492
|
+
end
|
493
|
+
|
494
|
+
watchers do |watcher|
|
495
|
+
login = watcher[:login].presence || generate_anonymous
|
496
|
+
community.total.watchers << login
|
497
|
+
community.total.users_involved << login
|
498
|
+
end
|
499
|
+
|
500
|
+
forkers do |forker|
|
501
|
+
community.total.forks << forker[:owner][:login]
|
502
|
+
community.total.users_involved << forker[:owner][:login]
|
503
|
+
community.quarters[forker[:created_at]].forks << forker[:owner][:login]
|
504
|
+
community.quarters[forker[:created_at]].users_involved << forker[:owner][:login]
|
505
|
+
end
|
506
|
+
rescue Octokit::NotFound => e
|
507
|
+
project.github_alias = NO_GITHUB_NAME
|
508
|
+
puts "Github NotFound Error: #{e.inspect}"
|
509
|
+
rescue Octokit::InvalidRepository => e
|
510
|
+
puts "Github InvalidRepository Error: #{e.inspect}"
|
511
|
+
end
|
512
|
+
|
513
|
+
MAX_ATTEMPTS = 5
|
514
|
+
|
515
|
+
def retry_call
|
516
|
+
attempt = 0
|
517
|
+
begin
|
518
|
+
yield
|
519
|
+
rescue Octokit::InternalServerError => e
|
520
|
+
attempt += 1
|
521
|
+
raise if attempt > MAX_ATTEMPTS
|
522
|
+
puts "Github Error: #{e.inspect}... retrying"
|
523
|
+
sleep(attempt * 5.minutes)
|
524
|
+
retry
|
525
|
+
end
|
526
|
+
end
|
527
|
+
|
528
|
+
# GitHub sometimes hides login, this is fallback
|
529
|
+
def generate_anonymous
|
530
|
+
@anonymous_count ||= 0
|
531
|
+
@anonymous_count += 1
|
532
|
+
"anonymous_#{@anonymous_count}"
|
533
|
+
end
|
534
|
+
end
|
535
|
+
end
|
536
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Ossert
|
3
|
+
module Fetch
|
4
|
+
class Rubygems
|
5
|
+
attr_reader :client, :project
|
6
|
+
|
7
|
+
extend Forwardable
|
8
|
+
def_delegators :project, :agility, :community, :meta
|
9
|
+
|
10
|
+
def initialize(project)
|
11
|
+
@client = SimpleClient.new('https://rubygems.org/api/v1/')
|
12
|
+
@project = project
|
13
|
+
end
|
14
|
+
|
15
|
+
def info
|
16
|
+
@info ||= client.get("gems/#{project.rubygems_alias}.json")
|
17
|
+
end
|
18
|
+
|
19
|
+
def version_info
|
20
|
+
@info ||= client.get("versions/#{project.rubygems_alias}.json")
|
21
|
+
end
|
22
|
+
|
23
|
+
def releases
|
24
|
+
@releases ||= client.get("versions/#{project.rubygems_alias}.json")
|
25
|
+
end
|
26
|
+
|
27
|
+
def reversed_dependencies
|
28
|
+
client.get("/gems/#{project.rubygems_alias}/reverse_dependencies.json")
|
29
|
+
end
|
30
|
+
|
31
|
+
def process_meta
|
32
|
+
meta[:authors] = info['authors']
|
33
|
+
meta[:description] = info['info']
|
34
|
+
meta[:current_version] = info['version']
|
35
|
+
end
|
36
|
+
|
37
|
+
def process_links
|
38
|
+
meta.merge!(
|
39
|
+
homepage_url: info['homepage_uri'],
|
40
|
+
docs_url: info['documentation_uri'],
|
41
|
+
wiki_url: info['wiki_uri'],
|
42
|
+
source_url: info['source_code_uri'],
|
43
|
+
issue_tracker_url: info['bug_tracker_uri'],
|
44
|
+
mailing_list_url: info['mailing_list_uri'],
|
45
|
+
rubygems_url: info['project_uri'],
|
46
|
+
github_url: "https://github.com/#{project.github_alias}" # or exception!
|
47
|
+
)
|
48
|
+
end
|
49
|
+
|
50
|
+
def process_github_alias
|
51
|
+
return unless project.github_alias.blank?
|
52
|
+
match = info['source_code_uri'].try(:match, %r{github.com/([a-zA-Z0-9\.\_\-]+)/([a-zA-Z0-9\.\_\-]+)})
|
53
|
+
match ||= info['homepage_uri'].try(:match, %r{github.com/([a-zA-Z0-9\.\_\-]+)/([a-zA-Z0-9\.\_\-]+)})
|
54
|
+
project.github_alias = match ? "#{match[1]}/#{match[2]}" : NO_GITHUB_NAME
|
55
|
+
end
|
56
|
+
|
57
|
+
def process_releases
|
58
|
+
releases.each do |release|
|
59
|
+
agility.total.releases_total_rg << release['number']
|
60
|
+
agility.quarters[release['created_at']].releases_total_rg << release['number']
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def process_dependencies
|
65
|
+
agility.total.dependencies = Set.new(info['dependencies']['runtime']).to_a
|
66
|
+
community.total.dependants = Set.new(reversed_dependencies).to_a
|
67
|
+
end
|
68
|
+
|
69
|
+
def process
|
70
|
+
process_github_alias
|
71
|
+
|
72
|
+
process_dependencies
|
73
|
+
process_releases
|
74
|
+
|
75
|
+
process_meta
|
76
|
+
process_links
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|