ossert 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/.rspec +2 -0
  4. data/.rubocop_todo.yml +44 -0
  5. data/.ruby-gemset +1 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +16 -0
  8. data/Gemfile +8 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +199 -0
  11. data/Rakefile +12 -0
  12. data/bin/console +14 -0
  13. data/bin/setup +8 -0
  14. data/config/classifiers.yml +153 -0
  15. data/config/descriptions.yml +45 -0
  16. data/config/sidekiq.rb +15 -0
  17. data/config/stats.yml +198 -0
  18. data/config/translations.yml +44 -0
  19. data/db/backups/.keep +0 -0
  20. data/db/migrate/001_create_projects.rb +22 -0
  21. data/db/migrate/002_create_exceptions.rb +14 -0
  22. data/db/migrate/003_add_meta_to_projects.rb +14 -0
  23. data/db/migrate/004_add_timestamps_to_projects.rb +12 -0
  24. data/db/migrate/005_create_classifiers.rb +19 -0
  25. data/lib/ossert/classifiers/decision_tree.rb +112 -0
  26. data/lib/ossert/classifiers/growing/check.rb +172 -0
  27. data/lib/ossert/classifiers/growing/classifier.rb +175 -0
  28. data/lib/ossert/classifiers/growing.rb +163 -0
  29. data/lib/ossert/classifiers.rb +14 -0
  30. data/lib/ossert/config.rb +24 -0
  31. data/lib/ossert/fetch/bestgems.rb +98 -0
  32. data/lib/ossert/fetch/github.rb +536 -0
  33. data/lib/ossert/fetch/rubygems.rb +80 -0
  34. data/lib/ossert/fetch.rb +142 -0
  35. data/lib/ossert/presenters/project.rb +202 -0
  36. data/lib/ossert/presenters/project_v2.rb +117 -0
  37. data/lib/ossert/presenters.rb +8 -0
  38. data/lib/ossert/project.rb +144 -0
  39. data/lib/ossert/quarters_store.rb +164 -0
  40. data/lib/ossert/rake_tasks.rb +6 -0
  41. data/lib/ossert/reference.rb +87 -0
  42. data/lib/ossert/repositories.rb +138 -0
  43. data/lib/ossert/saveable.rb +153 -0
  44. data/lib/ossert/stats/agility_quarter.rb +62 -0
  45. data/lib/ossert/stats/agility_total.rb +71 -0
  46. data/lib/ossert/stats/base.rb +113 -0
  47. data/lib/ossert/stats/community_quarter.rb +28 -0
  48. data/lib/ossert/stats/community_total.rb +24 -0
  49. data/lib/ossert/stats.rb +32 -0
  50. data/lib/ossert/tasks/database.rake +179 -0
  51. data/lib/ossert/tasks/ossert.rake +52 -0
  52. data/lib/ossert/version.rb +4 -0
  53. data/lib/ossert/workers/fetch.rb +21 -0
  54. data/lib/ossert/workers/fetch_bestgems_page.rb +32 -0
  55. data/lib/ossert/workers/refresh_fetch.rb +22 -0
  56. data/lib/ossert/workers/sync_rubygems.rb +0 -0
  57. data/lib/ossert/workers.rb +11 -0
  58. data/lib/ossert.rb +63 -0
  59. data/ossert.gemspec +47 -0
  60. metadata +396 -0
@@ -0,0 +1,536 @@
1
+ # frozen_string_literal: true
2
+ require 'octokit'
3
+
4
+ module Ossert
5
+ module Fetch
6
+ class GitHub
7
+ attr_reader :client, :project
8
+
9
+ extend Forwardable
10
+ def_delegators :project, :agility, :community, :meta
11
+
12
+ def initialize(project)
13
+ @client = ::Octokit::Client.new(access_token: ENV['GITHUB_TOKEN'])
14
+ client.default_media_type = 'application/vnd.github.v3.star+json'
15
+ client.auto_paginate = true
16
+
17
+ @project = project
18
+ raise ArgumentError unless (@repo_name = project.github_alias).present?
19
+ @owner = @repo_name.split('/')[0]
20
+ @requests_count = 0
21
+ end
22
+
23
+ # TODO: Add github search feature
24
+ # def find_repo(user)
25
+ # first_found = client.search_repos(project.name, language: :ruby, user: user)[:items].first
26
+ # first_found.try(:[], :full_name)
27
+ # end
28
+
29
+ def request(endpoint, *args)
30
+ first_response_data = client.paginate(url(endpoint, args.shift), *args) do |_, last_response|
31
+ last_response.data.each { |data| yield data }
32
+ end
33
+ first_response_data.each { |data| yield data }
34
+ end
35
+
36
+ def url(endpoint, repo_name)
37
+ path = case endpoint
38
+ when /issues_comments/
39
+ 'issues/comments'
40
+ when /pulls_comments/
41
+ 'pulls/comments'
42
+ else
43
+ endpoint
44
+ end
45
+ "#{Octokit::Repository.path repo_name}/#{path}"
46
+ end
47
+
48
+ def issues(&block)
49
+ request(:issues, @repo_name, state: :all, &block)
50
+ end
51
+
52
+ def issues_comments(&block)
53
+ request(:issues_comments, @repo_name, &block)
54
+ end
55
+
56
+ def pulls(&block)
57
+ # fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
58
+ # http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
59
+ # fetch comments and link with PR by "pull_request_url"
60
+ request(:pulls, @repo_name, state: :all, &block)
61
+ end
62
+
63
+ def pulls_comments(&block)
64
+ # fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
65
+ # http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
66
+ # fetch comments and link with PR by "pull_request_url"
67
+ request(:pulls_comments, @repo_name, &block)
68
+ end
69
+
70
+ def contributors(&block)
71
+ request(:contributors, @repo_name, anon: true, &block)
72
+ end
73
+
74
+ def stargazers(&block)
75
+ request(:stargazers, @repo_name, &block)
76
+ end
77
+
78
+ def watchers(&block)
79
+ request(:subscribers, @repo_name, &block)
80
+ end
81
+
82
+ def forkers(&block)
83
+ request(:forks, @repo_name, &block)
84
+ end
85
+
86
+ def branches(&block)
87
+ request(:branches, @repo_name, &block)
88
+ end
89
+
90
+ def tags(&block)
91
+ request(:tags, @repo_name, &block)
92
+ end
93
+
94
+ def commits(from, to, &block)
95
+ request(:commits, @repo_name, since: from, until: to, &block)
96
+ end
97
+
98
+ def last_year_commits
99
+ last_year_commits = []
100
+ retry_count = 3
101
+ while last_year_commits.blank? && retry_count.positive?
102
+ last_year_commits = client.commit_activity_stats(@repo_name)
103
+ if last_year_commits.blank?
104
+ sleep(15 * retry_count)
105
+ retry_count -= 1
106
+ end
107
+ end
108
+ last_year_commits
109
+ end
110
+
111
+ def top_contributors
112
+ client.contributors_stats(@repo_name, retry_timeout: 5, retry_wait: 5)
113
+ end
114
+
115
+ def commit(sha)
116
+ client.commit(@repo_name, sha)
117
+ end
118
+
119
+ def tag_info(sha)
120
+ client.tag(@repo_name, sha)
121
+ rescue Octokit::NotFound
122
+ false
123
+ end
124
+
125
+ def date_from_tag(sha)
126
+ tag_info = tag_info(sha)
127
+ return tag_info[:tagger][:date] if tag_info
128
+ value = commit(sha)[:commit][:committer][:date]
129
+ DateTime.new(*value.split('-').map(&:to_i)).to_i
130
+ end
131
+
132
+ def commits_since(date)
133
+ client.commits_since(@repo_name, date)
134
+ end
135
+
136
+ def latest_release
137
+ @latest_release ||= client.latest_release(@repo_name)
138
+ end
139
+
140
+ # Add class with processing types, e.g. top_contributors, commits and so on
141
+
142
+ def process_top_contributors
143
+ @top_contributors = (top_contributors || []).map { |contrib_data| contrib_data[:author][:login] }
144
+ @top_contributors.last(10).reverse.each do |login|
145
+ (meta[:top_10_contributors] ||= []) << "https://github.com/#{login}"
146
+ end
147
+ nil
148
+ end
149
+
150
+ def process_commits
151
+ last_year_commits.each do |week|
152
+ current_count = agility.total.last_year_commits.to_i
153
+ agility.total.last_year_commits = current_count + week['total']
154
+
155
+ current_quarter_count = agility.quarters[week['week']].commits.to_i
156
+ agility.quarters[week['week']].commits = current_quarter_count + week['total']
157
+ end
158
+ end
159
+
160
+ def process_last_release_date
161
+ latest_release_date = 0
162
+
163
+ tags do |tag|
164
+ tag_date = date_from_tag(tag[:commit][:sha])
165
+ latest_release_date = [latest_release_date, tag_date].max
166
+
167
+ agility.total.releases_total_gh << tag[:name]
168
+ agility.quarters[tag_date].releases_total_gh << tag[:name]
169
+ end
170
+
171
+ return if latest_release_date.zero?
172
+
173
+ agility.total.last_release_date = latest_release_date # wrong: last_release_commit[:commit][:committer][:date]
174
+ agility.total.commits_count_since_last_release = commits_since(Time.at(latest_release_date).utc).length
175
+ end
176
+
177
+ def process_quarters_issues_and_prs_processing_days
178
+ issues do |issue|
179
+ next if issue.key? :pull_request
180
+ next unless issue[:state] == 'closed'
181
+ next unless issue[:closed_at].present?
182
+ days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
183
+ (agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
184
+ end
185
+
186
+ pulls do |pull|
187
+ next unless pull[:state] == 'closed'
188
+ next unless pull[:closed_at].present?
189
+ days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
190
+ (agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
191
+ end
192
+ end
193
+
194
+ def process_actual_prs_and_issues
195
+ actual_prs = Set.new
196
+ actual_issues = Set.new
197
+ agility.quarters.each_sorted do |_quarter, data|
198
+ data.pr_actual = actual_prs.to_a
199
+ data.issues_actual = actual_issues.to_a
200
+
201
+ closed = Set.new(data.pr_closed + data.issues_closed)
202
+ actual_prs = Set.new(actual_prs + data.pr_open) - closed
203
+ actual_issues = Set.new(actual_issues + data.issues_open) - closed
204
+ end
205
+ end
206
+
207
+ def issue2pull_url(html_url)
208
+ html_url.gsub(
209
+ %r{https://github.com/(#{@repo_name})/pull/(\d+)},
210
+ 'https://api.github.com/repos/\2/pulls/\3'
211
+ )
212
+ end
213
+
214
+ def process_open_pull(pull)
215
+ agility.total.pr_open << pull[:url]
216
+ agility.quarters[pull[:created_at]].pr_open << pull[:url]
217
+ end
218
+
219
+ def process_closed_pull(pull)
220
+ agility.total.pr_closed << pull[:url]
221
+ agility.quarters[pull[:created_at]].pr_open << pull[:url]
222
+ agility.quarters[pull[:closed_at]].pr_closed << pull[:url] if pull[:closed_at]
223
+ agility.quarters[pull[:merged_at]].pr_merged << pull[:url] if pull[:merged_at]
224
+
225
+ return unless pull[:closed_at].present?
226
+ days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
227
+ @pulls_processed_in_days << days_to_close
228
+ (agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
229
+ end
230
+
231
+ def process_users_from_pull(pull)
232
+ community.total.users_creating_pr << pull[:user][:login]
233
+ community.quarters[pull[:created_at]].users_creating_pr << pull[:user][:login]
234
+ community.total.users_involved << pull[:user][:login]
235
+ community.quarters[pull[:created_at]].users_involved << pull[:user][:login]
236
+ end
237
+
238
+ def process_pulls
239
+ @pulls_processed_in_days = Set.new
240
+
241
+ retry_call do
242
+ pulls do |pull|
243
+ case pull[:state]
244
+ when 'open'
245
+ process_open_pull(pull)
246
+ when 'closed'
247
+ process_closed_pull(pull)
248
+ end
249
+
250
+ if pull[:user][:login] == @owner
251
+ agility.total.pr_owner << pull[:url]
252
+ else
253
+ agility.total.pr_non_owner << pull[:url]
254
+ end
255
+
256
+ agility.total.pr_total << pull[:url]
257
+ agility.quarters[pull[:created_at]].pr_total << pull[:url]
258
+
259
+ created_at = pull[:created_at].to_datetime.to_i
260
+ if agility.total.first_pr_date.zero? || created_at < agility.total.first_pr_date
261
+ agility.total.first_pr_date = created_at
262
+ end
263
+
264
+ if agility.total.last_pr_date.zero? || created_at > agility.total.last_pr_date
265
+ agility.total.last_pr_date = created_at
266
+ end
267
+
268
+ process_users_from_pull(pull)
269
+ end
270
+ end
271
+
272
+ values = @pulls_processed_in_days.to_a.sort
273
+ agility.total.pr_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
274
+ agility.total.pr_processed_in_median = if values.count.odd?
275
+ values[values.count / 2]
276
+ elsif values.count.zero?
277
+ 0
278
+ else
279
+ ((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
280
+ end
281
+
282
+
283
+ retry_call do
284
+ pulls_comments do |pull_comment|
285
+ login = pull_comment[:user].try(:[], :login).presence || generate_anonymous
286
+ if community.total.contributors.include? login
287
+ agility.total.pr_with_contrib_comments << pull_comment[:pull_request_url]
288
+ end
289
+
290
+ community.total.users_commenting_pr << login
291
+ community.quarters[pull_comment[:created_at]].users_commenting_pr << login
292
+ community.total.users_involved << login
293
+ community.quarters[pull_comment[:created_at]].users_involved << login
294
+ end
295
+ end
296
+ end
297
+
298
+ def process_open_issue(issue)
299
+ agility.total.issues_open << issue[:url]
300
+ agility.quarters[issue[:created_at]].issues_open << issue[:url]
301
+ end
302
+
303
+ def process_closed_issue(issue)
304
+ agility.total.issues_closed << issue[:url]
305
+ # if issue is closed for now, it also was opened somewhen
306
+ agility.quarters[issue[:created_at]].issues_open << issue[:url]
307
+ agility.quarters[issue[:closed_at]].issues_closed << issue[:url] if issue[:closed_at]
308
+
309
+ return unless issue[:closed_at].present?
310
+ days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
311
+ @issues_processed_in_days << days_to_close
312
+ (agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
313
+ end
314
+
315
+ def process_users_from_issue(issue)
316
+ community.total.users_creating_issues << issue[:user][:login]
317
+ community.quarters[issue[:created_at]].users_creating_issues << issue[:user][:login]
318
+ community.total.users_involved << issue[:user][:login]
319
+ community.quarters[issue[:created_at]].users_involved << issue[:user][:login]
320
+ end
321
+
322
+ def process_issues
323
+ @issues_processed_in_days = []
324
+
325
+ issues do |issue|
326
+ next if issue.key? :pull_request
327
+ case issue[:state]
328
+ when 'open'
329
+ process_open_issue(issue)
330
+ when 'closed'
331
+ process_closed_issue(issue)
332
+ end
333
+
334
+ if issue[:user][:login] == @owner
335
+ agility.total.issues_owner << issue[:url]
336
+ else
337
+ agility.total.issues_non_owner << issue[:url]
338
+ end
339
+
340
+ agility.total.issues_total << issue[:url]
341
+ agility.quarters[issue[:created_at]].issues_total << issue[:url]
342
+
343
+ created_at = issue[:created_at].to_datetime.to_i
344
+ if agility.total.first_issue_date.zero? || created_at < agility.total.first_issue_date
345
+ agility.total.first_issue_date = created_at
346
+ end
347
+
348
+ if agility.total.last_issue_date.zero? || created_at > agility.total.last_issue_date
349
+ agility.total.last_issue_date = created_at
350
+ end
351
+
352
+ process_users_from_issue(issue)
353
+ end
354
+
355
+ values = @issues_processed_in_days.to_a.sort
356
+ agility.total.issues_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
357
+ agility.total.issues_processed_in_median = if values.count.odd?
358
+ values[values.count / 2]
359
+ elsif values.count.zero?
360
+ 0
361
+ else
362
+ ((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
363
+ end
364
+
365
+ issues_comments do |issue_comment|
366
+ login = issue_comment[:user].try(:[], :login).presence || generate_anonymous
367
+ issue_url = /\A(.*)#issuecomment.*\z/.match(issue_comment[:html_url])[1]
368
+ if issue_url.include?('/pull/') # PR comments are stored as Issue comments. Sadness =(
369
+ if community.total.contributors.include? login
370
+ agility.total.pr_with_contrib_comments << issue2pull_url(issue_url)
371
+ end
372
+
373
+ community.total.users_commenting_pr << login
374
+ community.quarters[issue_comment[:created_at]].users_commenting_pr << login
375
+ community.total.users_involved << login
376
+ community.quarters[issue_comment[:created_at]].users_involved << login
377
+ next
378
+ end
379
+
380
+ if community.total.contributors.include? login
381
+ agility.total.issues_with_contrib_comments << issue_url
382
+ end
383
+
384
+ community.total.users_commenting_issues << login
385
+ community.quarters[issue_comment[:created_at]].users_commenting_issues << login
386
+ community.total.users_involved << login
387
+ community.quarters[issue_comment[:created_at]].users_involved << login
388
+ end
389
+ end
390
+
391
+ def process
392
+ contributors do |c|
393
+ login = c.try(:[], :login).presence || generate_anonymous
394
+ community.total.contributors << login
395
+ end
396
+ community.total.users_involved += community.total.contributors
397
+ community.total.users_involved.uniq!
398
+
399
+ # TODO: extract contributors and commits, quarter by quarter.
400
+ #
401
+ # => {:sha=>"d1a43d32e615b4a75117151b002266c560ce9061",
402
+ # :commit=>
403
+ # {:author=>
404
+ # {:name=>"Yves Senn",
405
+ # :email=>"yves.senn@gmail.com",
406
+ # :date=>"2015-09-22T08:25:14Z"},
407
+ # :committer=>
408
+ # {:name=>"Yves Senn",
409
+ # :email=>"yves.senn@gmail.com",
410
+ # :date=>"2015-09-22T08:25:14Z"},
411
+ # :message=>
412
+ # "Merge pull request #21678 from ronakjangir47/array_to_formatted_s_docs\n\nAdded Examples in docs for int
413
+ # ernal behavior of Array#to_formatted_s [ci skip]",
414
+ # :tree=>
415
+ # {:sha=>"204811aa155645b461467dbd2238ac41c0fe8a30",
416
+ # :url=>
417
+ # "https://api.github.com/repos/rails/rails/git/trees/204811aa155645b461467dbd2238ac41c0fe8a30"},
418
+ # :url=>
419
+ # "https://api.github.com/repos/rails/rails/git/commits/d1a43d32e615b4a75117151b002266c560ce9061",
420
+ # :comment_count=>0},
421
+ # :url=>
422
+ # "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061",
423
+ # :html_url=>
424
+ # "https://github.com/rails/rails/commit/d1a43d32e615b4a75117151b002266c560ce9061",
425
+ # :comments_url=>
426
+ # "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061/comments",
427
+ # :author=>
428
+ # {:login=>"senny",
429
+ # ...
430
+ # :type=>"User",
431
+ # :site_admin=>false},
432
+ # :committer=>
433
+ # {:login=>"senny",
434
+ # ...
435
+ # :type=>"User",
436
+ # :site_admin=>false},
437
+ # :parents=>
438
+ # [{:sha=>"2a7e8f54c66dbd65822f2a7135546a240426b631",
439
+ # :url=>
440
+ # "https://api.github.com/repos/rails/rails/commits/2a7e8f54c66dbd65822f2a7135546a240426b631",
441
+ # :html_url=>
442
+ # "https://github.com/rails/rails/commit/2a7e8f54c66dbd65822f2a7135546a240426b631"},
443
+ # {:sha=>"192d29f1c7ea16c506c09da2b854d1acdfbc8749",
444
+ # :url=>
445
+ # "https://api.github.com/repos/rails/rails/commits/192d29f1c7ea16c506c09da2b854d1acdfbc8749",
446
+ # :html_url=>
447
+ # "https://github.com/rails/rails/commit/192d29f1c7ea16c506c09da2b854d1acdfbc8749"}]}
448
+
449
+ # process collaborators and commits. year by year, more info then ^^^^^
450
+ # count = 0; collab = Set.new; fetcher.commits(14.months.ago.utc.iso8601, 13.month.ago.utc.iso8601) do |commit|
451
+ # count+=1
452
+ # collab << (commit[:author].try(:[],:login) || commit[:commit][:author][:name])
453
+ # end
454
+
455
+ process_issues
456
+
457
+ process_pulls
458
+
459
+ process_actual_prs_and_issues
460
+
461
+ process_last_release_date
462
+
463
+ process_commits
464
+
465
+ sleep(1)
466
+
467
+ process_top_contributors
468
+
469
+ sleep(1)
470
+
471
+ branches do |branch|
472
+ # stale and total
473
+ # by quarter ? date from commit -> [:commit][:committer][:date]
474
+ # 1. save dates by commit sha.
475
+ branch_updated_at = commit(branch[:commit][:sha])[:commit][:committer][:date]
476
+ stale_threshold = Time.now.beginning_of_quarter
477
+
478
+ # 2. date -> total by quarter
479
+ # date -> stale
480
+ agility.total.branches << branch[:name]
481
+ agility.total.stale_branches << branch[:name] if branch_updated_at < stale_threshold
482
+ agility.quarters[branch_updated_at].branches << branch[:name]
483
+ end
484
+
485
+ stargazers do |stargazer|
486
+ login = stargazer[:user][:login].presence || generate_anonymous
487
+ community.total.stargazers << login
488
+ community.total.users_involved << login
489
+
490
+ community.quarters[stargazer[:starred_at]].stargazers << login
491
+ community.quarters[stargazer[:starred_at]].users_involved << login
492
+ end
493
+
494
+ watchers do |watcher|
495
+ login = watcher[:login].presence || generate_anonymous
496
+ community.total.watchers << login
497
+ community.total.users_involved << login
498
+ end
499
+
500
+ forkers do |forker|
501
+ community.total.forks << forker[:owner][:login]
502
+ community.total.users_involved << forker[:owner][:login]
503
+ community.quarters[forker[:created_at]].forks << forker[:owner][:login]
504
+ community.quarters[forker[:created_at]].users_involved << forker[:owner][:login]
505
+ end
506
+ rescue Octokit::NotFound => e
507
+ project.github_alias = NO_GITHUB_NAME
508
+ puts "Github NotFound Error: #{e.inspect}"
509
+ rescue Octokit::InvalidRepository => e
510
+ puts "Github InvalidRepository Error: #{e.inspect}"
511
+ end
512
+
513
+ MAX_ATTEMPTS = 5
514
+
515
+ def retry_call
516
+ attempt = 0
517
+ begin
518
+ yield
519
+ rescue Octokit::InternalServerError => e
520
+ attempt += 1
521
+ raise if attempt > MAX_ATTEMPTS
522
+ puts "Github Error: #{e.inspect}... retrying"
523
+ sleep(attempt * 5.minutes)
524
+ retry
525
+ end
526
+ end
527
+
528
+ # GitHub sometimes hides login, this is fallback
529
+ def generate_anonymous
530
+ @anonymous_count ||= 0
531
+ @anonymous_count += 1
532
+ "anonymous_#{@anonymous_count}"
533
+ end
534
+ end
535
+ end
536
+ end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+ module Ossert
3
+ module Fetch
4
+ class Rubygems
5
+ attr_reader :client, :project
6
+
7
+ extend Forwardable
8
+ def_delegators :project, :agility, :community, :meta
9
+
10
+ def initialize(project)
11
+ @client = SimpleClient.new('https://rubygems.org/api/v1/')
12
+ @project = project
13
+ end
14
+
15
+ def info
16
+ @info ||= client.get("gems/#{project.rubygems_alias}.json")
17
+ end
18
+
19
+ def version_info
20
+ @info ||= client.get("versions/#{project.rubygems_alias}.json")
21
+ end
22
+
23
+ def releases
24
+ @releases ||= client.get("versions/#{project.rubygems_alias}.json")
25
+ end
26
+
27
+ def reversed_dependencies
28
+ client.get("/gems/#{project.rubygems_alias}/reverse_dependencies.json")
29
+ end
30
+
31
+ def process_meta
32
+ meta[:authors] = info['authors']
33
+ meta[:description] = info['info']
34
+ meta[:current_version] = info['version']
35
+ end
36
+
37
+ def process_links
38
+ meta.merge!(
39
+ homepage_url: info['homepage_uri'],
40
+ docs_url: info['documentation_uri'],
41
+ wiki_url: info['wiki_uri'],
42
+ source_url: info['source_code_uri'],
43
+ issue_tracker_url: info['bug_tracker_uri'],
44
+ mailing_list_url: info['mailing_list_uri'],
45
+ rubygems_url: info['project_uri'],
46
+ github_url: "https://github.com/#{project.github_alias}" # or exception!
47
+ )
48
+ end
49
+
50
+ def process_github_alias
51
+ return unless project.github_alias.blank?
52
+ match = info['source_code_uri'].try(:match, %r{github.com/([a-zA-Z0-9\.\_\-]+)/([a-zA-Z0-9\.\_\-]+)})
53
+ match ||= info['homepage_uri'].try(:match, %r{github.com/([a-zA-Z0-9\.\_\-]+)/([a-zA-Z0-9\.\_\-]+)})
54
+ project.github_alias = match ? "#{match[1]}/#{match[2]}" : NO_GITHUB_NAME
55
+ end
56
+
57
+ def process_releases
58
+ releases.each do |release|
59
+ agility.total.releases_total_rg << release['number']
60
+ agility.quarters[release['created_at']].releases_total_rg << release['number']
61
+ end
62
+ end
63
+
64
+ def process_dependencies
65
+ agility.total.dependencies = Set.new(info['dependencies']['runtime']).to_a
66
+ community.total.dependants = Set.new(reversed_dependencies).to_a
67
+ end
68
+
69
+ def process
70
+ process_github_alias
71
+
72
+ process_dependencies
73
+ process_releases
74
+
75
+ process_meta
76
+ process_links
77
+ end
78
+ end
79
+ end
80
+ end