jirametrics 2.30.1pre1 → 2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/jirametrics/download_config.rb +1 -5
- data/lib/jirametrics/downloader_for_cloud.rb +113 -20
- data/lib/jirametrics/downloader_for_data_center.rb +35 -1
- data/lib/jirametrics/file_system.rb +4 -0
- data/lib/jirametrics/github_gateway.rb +123 -22
- data/lib/jirametrics/jira_gateway.rb +16 -2
- data/lib/jirametrics/project_config.rb +16 -11
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e5d255e03d686d0bf827ab0cd09191777b9586574971d6e274704a85df1059a6
|
|
4
|
+
data.tar.gz: 60fbf1432b7399186a5e854ef58e83ee59ae96b5bee859371b09f992b4854db7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 79003fc0ab19a3253e9611c74e5d5772b7e0cf17250ec668c9c21cf871056020fa1d6af979387b4e76395b47a687e300b288555d0709c9fa50a7c227d359dbe2
|
|
7
|
+
data.tar.gz: 7587f1a620a1015f174bfa5b2b2e31251f3b0c0e2be0a688aa31e1da423bdefa2deaab7a8d40c4e3a8fd54f8c495a3b43db1a0ad231feab6624d8cf74b08adff
|
|
@@ -30,7 +30,7 @@ class DownloadConfig
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def github_repo *repos
|
|
33
|
-
github_repos.concat(repos
|
|
33
|
+
github_repos.concat(repos)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def start_date today:
|
|
@@ -42,8 +42,4 @@ class DownloadConfig
|
|
|
42
42
|
|
|
43
43
|
private
|
|
44
44
|
|
|
45
|
-
def normalize_github_repo repo
|
|
46
|
-
match = repo.match(%r{github\.com/([^/]+/[^/]+?)/?$})
|
|
47
|
-
match ? match[1] : repo
|
|
48
|
-
end
|
|
49
45
|
end
|
|
@@ -101,6 +101,7 @@ class DownloaderForCloud < Downloader
|
|
|
101
101
|
)
|
|
102
102
|
|
|
103
103
|
attach_changelog_to_issues issue_datas: issue_datas, issue_jsons: response['issues']
|
|
104
|
+
attach_worklogs_to_issues issue_datas: issue_datas, issue_jsons: response['issues']
|
|
104
105
|
|
|
105
106
|
response['issues'].each do |issue_json|
|
|
106
107
|
issue_json['exporter'] = {
|
|
@@ -129,6 +130,49 @@ class DownloaderForCloud < Downloader
|
|
|
129
130
|
issue_datas
|
|
130
131
|
end
|
|
131
132
|
|
|
133
|
+
def attach_worklogs_to_issues issue_datas:, issue_jsons:, max_results: 100 # rubocop:disable Lint/UnusedMethodArgument
|
|
134
|
+
issue_jsons.each do |issue_json|
|
|
135
|
+
worklog = issue_json['fields']['worklog']
|
|
136
|
+
next unless worklog
|
|
137
|
+
|
|
138
|
+
total = worklog['total'].to_i
|
|
139
|
+
all_worklogs = worklog['worklogs'] || []
|
|
140
|
+
next if all_worklogs.size >= total
|
|
141
|
+
|
|
142
|
+
key = issue_json['key']
|
|
143
|
+
start_at = all_worklogs.size
|
|
144
|
+
|
|
145
|
+
loop do
|
|
146
|
+
response = @jira_gateway.call_url(
|
|
147
|
+
relative_url: "/rest/api/3/issue/#{CGI.escape(key)}/worklog?startAt=#{start_at}&maxResults=#{max_results}"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
worklogs = response['worklogs'] || []
|
|
151
|
+
all_worklogs.concat(worklogs)
|
|
152
|
+
|
|
153
|
+
total = response['total'].to_i
|
|
154
|
+
log " #{key} worklogs: page startAt=#{start_at}, " \
|
|
155
|
+
"received=#{worklogs.size}, fetched=#{all_worklogs.size}/#{total}"
|
|
156
|
+
break if all_worklogs.size >= total
|
|
157
|
+
# Guard against Jira reporting a higher total than it will actually return — seen when
|
|
158
|
+
# worklogs are deleted or access-restricted after the initial fetch. Without this,
|
|
159
|
+
# start_at never advances and we loop forever requesting the same empty page.
|
|
160
|
+
break if worklogs.empty?
|
|
161
|
+
|
|
162
|
+
start_at += worklogs.size
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
issue_json['fields']['worklog'] = {
|
|
166
|
+
'startAt' => 0,
|
|
167
|
+
'maxResults' => all_worklogs.size,
|
|
168
|
+
'total' => all_worklogs.size,
|
|
169
|
+
'worklogs' => all_worklogs
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
log " Enhanced #{key} with #{all_worklogs.size} worklogs"
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
132
176
|
def attach_changelog_to_issues issue_datas:, issue_jsons:
|
|
133
177
|
max_results = 10_000 # The max jira accepts is 10K
|
|
134
178
|
payload = {
|
|
@@ -187,10 +231,14 @@ class DownloaderForCloud < Downloader
|
|
|
187
231
|
loop do
|
|
188
232
|
related_issue_keys = Set.new
|
|
189
233
|
stale = issue_data_hash.values.reject { |data| data.up_to_date }
|
|
234
|
+
if in_related_phase
|
|
235
|
+
@file_system.diagnostic "Download loop: #{issue_data_hash.size} total known, " \
|
|
236
|
+
"#{stale.size} stale, #{checked_for_related.size} link-scanned"
|
|
237
|
+
end
|
|
190
238
|
unless stale.empty?
|
|
191
239
|
log_start ' Downloading more issues ' unless in_related_phase
|
|
192
240
|
stale.each_slice(100) do |slice|
|
|
193
|
-
slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query:
|
|
241
|
+
slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query: !in_related_phase)
|
|
194
242
|
progress_dot
|
|
195
243
|
slice.each do |data|
|
|
196
244
|
next unless data.issue
|
|
@@ -202,23 +250,20 @@ class DownloaderForCloud < Downloader
|
|
|
202
250
|
# to parse the file just to find the timestamp
|
|
203
251
|
@file_system.utime time: data.issue.updated, file: data.cache_path
|
|
204
252
|
|
|
205
|
-
|
|
253
|
+
collect_or_log_related(
|
|
254
|
+
issue: data.issue, found_in_primary_query: data.found_in_primary_query,
|
|
255
|
+
related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash
|
|
256
|
+
)
|
|
206
257
|
checked_for_related << data.key
|
|
207
258
|
end
|
|
208
259
|
end
|
|
209
260
|
end_progress unless in_related_phase
|
|
210
261
|
end
|
|
211
262
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
next unless @file_system.file_exist?(data.cache_path)
|
|
217
|
-
|
|
218
|
-
checked_for_related << data.key
|
|
219
|
-
raw = @file_system.load_json(data.cache_path)
|
|
220
|
-
collect_related_issue_keys issue: Issue.new(raw: raw, board: board), related_issue_keys: related_issue_keys
|
|
221
|
-
end
|
|
263
|
+
scan_cached_issues_for_related(
|
|
264
|
+
issue_data_hash: issue_data_hash, board: board,
|
|
265
|
+
checked_for_related: checked_for_related, related_issue_keys: related_issue_keys
|
|
266
|
+
)
|
|
222
267
|
|
|
223
268
|
# Remove all the ones we already have
|
|
224
269
|
related_issue_keys.reject! { |key| issue_data_hash[key] }
|
|
@@ -232,11 +277,11 @@ class DownloaderForCloud < Downloader
|
|
|
232
277
|
end
|
|
233
278
|
break if related_issue_keys.empty?
|
|
234
279
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
280
|
+
next if in_related_phase
|
|
281
|
+
|
|
282
|
+
in_related_phase = true
|
|
283
|
+
log " Identifying related issues (parents, subtasks, links) for board #{board.id}", both: true
|
|
284
|
+
log_start ' Downloading more issues '
|
|
240
285
|
end
|
|
241
286
|
|
|
242
287
|
end_progress if in_related_phase
|
|
@@ -265,20 +310,68 @@ class DownloaderForCloud < Downloader
|
|
|
265
310
|
end
|
|
266
311
|
end
|
|
267
312
|
|
|
313
|
+
# Scan up-to-date cached primary issues we haven't checked yet — they may reference related
|
|
314
|
+
# issues that are not in the primary query result. We only follow links one hop out from the
|
|
315
|
+
# primary issues, so related (non-primary) cached issues are not followed (just logged).
|
|
316
|
+
def scan_cached_issues_for_related issue_data_hash:, board:, checked_for_related:, related_issue_keys:
|
|
317
|
+
issue_data_hash.each_value do |data|
|
|
318
|
+
next if checked_for_related.include?(data.key)
|
|
319
|
+
next unless @file_system.file_exist?(data.cache_path)
|
|
320
|
+
|
|
321
|
+
checked_for_related << data.key
|
|
322
|
+
issue = Issue.new(raw: @file_system.load_json(data.cache_path), board: board)
|
|
323
|
+
collect_or_log_related(
|
|
324
|
+
issue: issue, found_in_primary_query: data.found_in_primary_query,
|
|
325
|
+
related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash
|
|
326
|
+
)
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Follow links one hop out from primary issues; for related (non-primary) issues, log the
|
|
331
|
+
# onward links we are deliberately not following rather than recursing into them.
|
|
332
|
+
def collect_or_log_related issue:, found_in_primary_query:, related_issue_keys:, issue_data_hash:
|
|
333
|
+
if found_in_primary_query
|
|
334
|
+
collect_related_issue_keys issue: issue, related_issue_keys: related_issue_keys
|
|
335
|
+
else
|
|
336
|
+
log_unfollowed_related_keys issue: issue, issue_data_hash: issue_data_hash
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
|
|
268
340
|
def collect_related_issue_keys issue:, related_issue_keys:
|
|
341
|
+
related_issue_keys.merge related_keys_for(issue)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# The parents, subtasks, and (non-cloner) linked issues that this issue references.
|
|
345
|
+
def related_keys_for issue
|
|
346
|
+
keys = Set.new
|
|
347
|
+
|
|
269
348
|
parent_key = issue.parent_key(project_config: @download_config.project_config)
|
|
270
|
-
|
|
349
|
+
keys << parent_key if parent_key
|
|
271
350
|
|
|
272
351
|
issue.raw['fields']['subtasks']&.each do |raw_subtask|
|
|
273
|
-
|
|
352
|
+
keys << raw_subtask['key']
|
|
274
353
|
end
|
|
275
354
|
|
|
276
355
|
issue.raw['fields']['issuelinks']&.each do |link|
|
|
277
356
|
next if link['type']['name'] == 'Cloners'
|
|
278
357
|
|
|
279
358
|
linked = link['inwardIssue'] || link['outwardIssue']
|
|
280
|
-
|
|
359
|
+
keys << linked['key'] if linked
|
|
281
360
|
end
|
|
361
|
+
|
|
362
|
+
keys
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# We only follow links one hop out from the primary (board) issues. If a related issue
|
|
366
|
+
# itself references further issues we haven't already downloaded, we deliberately don't
|
|
367
|
+
# follow them — but log it so we can diagnose later if an export fails because a
|
|
368
|
+
# second-hop issue was missing. See GitHub #72.
|
|
369
|
+
def log_unfollowed_related_keys issue:, issue_data_hash:
|
|
370
|
+
onward = related_keys_for(issue).reject { |key| issue_data_hash[key] }
|
|
371
|
+
return if onward.empty?
|
|
372
|
+
|
|
373
|
+
@file_system.diagnostic "One-hop limit: not following #{onward.size} onward link(s) from related " \
|
|
374
|
+
"issue #{issue.key}: #{onward.to_a.sort.join(', ')}"
|
|
282
375
|
end
|
|
283
376
|
|
|
284
377
|
def last_modified filename:
|
|
@@ -49,8 +49,12 @@ class DownloaderForDataCenter < Downloader
|
|
|
49
49
|
}
|
|
50
50
|
identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
|
|
51
51
|
file = "#{issue_json['key']}-#{board.id}.json"
|
|
52
|
+
issue_path = File.join(path, file)
|
|
52
53
|
|
|
53
|
-
@file_system.save_json(json: issue_json, filename:
|
|
54
|
+
@file_system.save_json(json: issue_json, filename: issue_path)
|
|
55
|
+
|
|
56
|
+
# Fetch complete worklog data for this issue
|
|
57
|
+
enhance_issue_with_worklogs(issue_key: issue_json['key'], issue_path: issue_path)
|
|
54
58
|
end
|
|
55
59
|
|
|
56
60
|
total = json['total'].to_i
|
|
@@ -63,6 +67,36 @@ class DownloaderForDataCenter < Downloader
|
|
|
63
67
|
end
|
|
64
68
|
end
|
|
65
69
|
|
|
70
|
+
def enhance_issue_with_worklogs issue_key:, issue_path:, max_results: 100
|
|
71
|
+
all_worklogs = []
|
|
72
|
+
start_at = 0
|
|
73
|
+
|
|
74
|
+
loop do
|
|
75
|
+
url = "/rest/api/2/issue/#{CGI.escape(issue_key)}/worklog?startAt=#{start_at}&maxResults=#{max_results}"
|
|
76
|
+
response = @jira_gateway.call_url(relative_url: url)
|
|
77
|
+
|
|
78
|
+
worklogs = response['worklogs'] || []
|
|
79
|
+
all_worklogs.concat(worklogs)
|
|
80
|
+
|
|
81
|
+
total = response['total'].to_i
|
|
82
|
+
break if start_at + worklogs.size >= total
|
|
83
|
+
|
|
84
|
+
start_at += worklogs.size
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
issue_json = @file_system.load_json(issue_path)
|
|
88
|
+
issue_json['fields'] ||= {}
|
|
89
|
+
issue_json['fields']['worklog'] = {
|
|
90
|
+
'startAt' => 0,
|
|
91
|
+
'maxResults' => all_worklogs.size,
|
|
92
|
+
'total' => all_worklogs.size,
|
|
93
|
+
'worklogs' => all_worklogs
|
|
94
|
+
}
|
|
95
|
+
@file_system.save_json(json: issue_json, filename: issue_path)
|
|
96
|
+
|
|
97
|
+
log " Enhanced #{issue_key} with #{all_worklogs.size} worklogs" if all_worklogs.any?
|
|
98
|
+
end
|
|
99
|
+
|
|
66
100
|
def make_jql filter_id:, today: nil
|
|
67
101
|
today ||= today_in_project_timezone
|
|
68
102
|
segments = []
|
|
@@ -55,6 +55,10 @@ class FileSystem
|
|
|
55
55
|
log "Error: #{message}", more: more, also_write_to_stderr: true
|
|
56
56
|
end
|
|
57
57
|
|
|
58
|
+
def diagnostic message, more: nil
|
|
59
|
+
log " [diag] #{message}", more: more
|
|
60
|
+
end
|
|
61
|
+
|
|
58
62
|
def log message, more: nil, also_write_to_stderr: false
|
|
59
63
|
message += " See #{logfile_name} for more details about this message." if more
|
|
60
64
|
|
|
@@ -6,6 +6,18 @@ require 'json'
|
|
|
6
6
|
class GithubGateway
|
|
7
7
|
attr_reader :repo
|
|
8
8
|
|
|
9
|
+
TRANSIENT_ERROR_PATTERNS = (
|
|
10
|
+
[429, 500, 502, 503, 504].map { |code| "HTTP #{code}" } +
|
|
11
|
+
['stream error:', 'unexpected end of JSON input']
|
|
12
|
+
).freeze
|
|
13
|
+
MAX_RETRIES = 3
|
|
14
|
+
REVIEW_STATES = %w[APPROVED CHANGES_REQUESTED].freeze
|
|
15
|
+
|
|
16
|
+
# How many keyless PRs to request commits for in a single graphql call, and how many commits
|
|
17
|
+
# to pull per PR. Kept bounded so node counts stay well under GitHub's graphql node limit.
|
|
18
|
+
COMMIT_FETCH_BATCH_SIZE = 30
|
|
19
|
+
MAX_COMMITS_PER_PR = 100
|
|
20
|
+
|
|
9
21
|
def initialize repo:, project_keys:, file_system:, raw_pr_cache: {}
|
|
10
22
|
@repo = repo
|
|
11
23
|
@project_keys = project_keys
|
|
@@ -16,11 +28,12 @@ class GithubGateway
|
|
|
16
28
|
|
|
17
29
|
def fetch_pull_requests since: nil
|
|
18
30
|
raw_prs = @raw_pr_cache[[@repo, since]] ||= fetch_raw_pull_requests(since: since)
|
|
31
|
+
prefetch_commit_messages(raw_prs)
|
|
19
32
|
raw_prs.filter_map { |pr| build_pr_data(pr) }
|
|
20
33
|
end
|
|
21
34
|
|
|
22
35
|
def fetch_raw_pull_requests since: nil
|
|
23
|
-
#
|
|
36
|
+
# NOTE: 'commits' is intentionally excluded — including it triggers GitHub's GraphQL node
|
|
24
37
|
# limit (authors sub-connection × PRs × commits exceeds 500,000 nodes). Branch name,
|
|
25
38
|
# title, and body are sufficient for issue key extraction in the vast majority of cases.
|
|
26
39
|
json_fields = %w[number title body headRefName createdAt closedAt mergedAt
|
|
@@ -58,21 +71,20 @@ class GithubGateway
|
|
|
58
71
|
def extract_issue_keys raw_pr
|
|
59
72
|
return [] if @issue_key_pattern.nil?
|
|
60
73
|
|
|
61
|
-
|
|
62
|
-
raw_pr['headRefName'],
|
|
63
|
-
raw_pr['title'],
|
|
64
|
-
raw_pr['body']
|
|
65
|
-
]
|
|
66
|
-
|
|
67
|
-
keys = sources.compact.flat_map { |s| s.scan(@issue_key_pattern) }.uniq
|
|
74
|
+
keys = keys_from_text_fields(raw_pr)
|
|
68
75
|
return keys unless keys.empty?
|
|
69
76
|
|
|
70
77
|
commit_messages_for(raw_pr['number']).flat_map { |msg| msg.scan(@issue_key_pattern) }.uniq
|
|
71
78
|
end
|
|
72
79
|
|
|
80
|
+
def keys_from_text_fields raw_pr
|
|
81
|
+
sources = [raw_pr['headRefName'], raw_pr['title'], raw_pr['body']]
|
|
82
|
+
sources.compact.flat_map { |s| s.scan(@issue_key_pattern) }.uniq
|
|
83
|
+
end
|
|
84
|
+
|
|
73
85
|
def extract_reviews raw_reviews
|
|
74
86
|
raw_reviews
|
|
75
|
-
.select { |r|
|
|
87
|
+
.select { |r| REVIEW_STATES.include?(r['state']) }
|
|
76
88
|
.map do |r|
|
|
77
89
|
{
|
|
78
90
|
'author' => r.dig('author', 'login'),
|
|
@@ -84,11 +96,73 @@ class GithubGateway
|
|
|
84
96
|
|
|
85
97
|
private
|
|
86
98
|
|
|
99
|
+
# Pre-populate the shared commit cache for every PR with no key in its branch/title/body, using
|
|
100
|
+
# batched graphql requests instead of one "gh pr view" per PR. build_pr_data -> commit_messages_for
|
|
101
|
+
# then reads straight from the cache for those PRs (no per-PR network call). Any PR the batch can't
|
|
102
|
+
# fully cover (more commits than one page, or absent from the response) is left uncached so the
|
|
103
|
+
# single-PR fallback in commit_messages_for fills it in.
|
|
104
|
+
def prefetch_commit_messages raw_prs
|
|
105
|
+
return if @issue_key_pattern.nil?
|
|
106
|
+
|
|
107
|
+
numbers = raw_prs
|
|
108
|
+
.select { |raw_pr| keys_from_text_fields(raw_pr).empty? }
|
|
109
|
+
.map { |raw_pr| raw_pr['number'] }
|
|
110
|
+
.reject { |number| @raw_pr_cache.key?([@repo, :commits, number]) }
|
|
111
|
+
|
|
112
|
+
numbers.each_slice(COMMIT_FETCH_BATCH_SIZE) do |batch|
|
|
113
|
+
fetch_commits_batch(batch).each do |number, messages|
|
|
114
|
+
@raw_pr_cache[[@repo, :commits, number]] = messages
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def fetch_commits_batch numbers
|
|
120
|
+
owner, name = owner_and_name
|
|
121
|
+
aliases = numbers.each_with_index.map do |number, index|
|
|
122
|
+
"pr#{index}: pullRequest(number: #{number}) " \
|
|
123
|
+
"{ commits(first: #{MAX_COMMITS_PER_PR}) { totalCount nodes { commit { messageHeadline messageBody } } } }"
|
|
124
|
+
end
|
|
125
|
+
query = %(query { repository(owner: "#{owner}", name: "#{name}") { #{aliases.join(' ')} } })
|
|
126
|
+
result = run_command(['api', 'graphql', '-f', "query=#{query}"])
|
|
127
|
+
parse_commits_batch result: result, numbers: numbers
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def parse_commits_batch result:, numbers:
|
|
131
|
+
repository = result.dig('data', 'repository') || {}
|
|
132
|
+
messages_by_number = {}
|
|
133
|
+
numbers.each_with_index do |number, index|
|
|
134
|
+
commits = repository.dig("pr#{index}", 'commits')
|
|
135
|
+
next if commits.nil?
|
|
136
|
+
|
|
137
|
+
nodes = commits['nodes'] || []
|
|
138
|
+
# Skip caching when the PR has more commits than this page covers, so the single-PR
|
|
139
|
+
# fallback fetches the complete set rather than us caching a partial answer.
|
|
140
|
+
next if commits['totalCount'] && commits['totalCount'] > nodes.size
|
|
141
|
+
|
|
142
|
+
messages_by_number[number] = nodes.flat_map do |node|
|
|
143
|
+
commit = node['commit'] || {}
|
|
144
|
+
[commit['messageHeadline'], commit['messageBody']].compact
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
messages_by_number
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def owner_and_name
|
|
151
|
+
# @repo may be a full URL (https://github.com/owner/name.git) or an owner/name slug.
|
|
152
|
+
@repo.sub(%r{\Ahttps?://[^/]+/}, '').delete_suffix('.git').split('/', 2)
|
|
153
|
+
end
|
|
154
|
+
|
|
87
155
|
def commit_messages_for pr_number
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
156
|
+
# Cached in the shared per-run cache (keyed by repo + PR) so the fallback isn't re-fetched
|
|
157
|
+
# when the same repo is downloaded by more than one project. Commit text doesn't depend on
|
|
158
|
+
# project_keys, so it's safe to share across projects with different keys. prefetch_commit_messages
|
|
159
|
+
# normally fills this in via a batched request; this single-PR path is the fallback.
|
|
160
|
+
@raw_pr_cache[[@repo, :commits, pr_number]] ||= begin
|
|
161
|
+
args = ['pr', 'view', pr_number.to_s, '--json', 'commits', '--repo', @repo]
|
|
162
|
+
result = run_command(args)
|
|
163
|
+
(result['commits'] || []).flat_map do |commit|
|
|
164
|
+
[commit['messageHeadline'], commit['messageBody']].compact
|
|
165
|
+
end
|
|
92
166
|
end
|
|
93
167
|
end
|
|
94
168
|
|
|
@@ -99,17 +173,44 @@ class GithubGateway
|
|
|
99
173
|
Regexp.new("\\b(?:#{keys_pattern})-\\d+(?![A-Za-z0-9])")
|
|
100
174
|
end
|
|
101
175
|
|
|
102
|
-
def
|
|
103
|
-
|
|
176
|
+
def monotonic_time
|
|
177
|
+
# In its own method so we can mock it out in tests
|
|
178
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
179
|
+
end
|
|
104
180
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
181
|
+
def run_command args
|
|
182
|
+
attempts = 0
|
|
183
|
+
loop do
|
|
184
|
+
attempts += 1
|
|
185
|
+
started = monotonic_time
|
|
186
|
+
stdout, stderr, status = Open3.capture3('gh', *args)
|
|
187
|
+
@file_system.diagnostic "gh #{args.first(2).join(' ')} call took #{format('%.2f', monotonic_time - started)}s"
|
|
188
|
+
|
|
189
|
+
# This extra check seems to only matter on Windows. On the mac, auth failures don't pass status.success?
|
|
190
|
+
if stderr.include?('SAML enforcement')
|
|
191
|
+
raise "GitHub CLI is not authorized to access #{@repo}. " \
|
|
192
|
+
'Run: gh auth refresh -h github.com -s read:org'
|
|
193
|
+
end
|
|
110
194
|
|
|
111
|
-
|
|
195
|
+
unless status.success?
|
|
196
|
+
error_message = " GitHub CLI command failed for #{@repo} " \
|
|
197
|
+
"(attempt #{attempts}/#{MAX_RETRIES}): #{stderr.strip}"
|
|
198
|
+
if attempts < MAX_RETRIES && TRANSIENT_ERROR_PATTERNS.any? { |pattern| stderr.include?(pattern) }
|
|
199
|
+
delay = 2**attempts
|
|
200
|
+
@file_system.log error_message
|
|
201
|
+
@file_system.log " Transient error detected. Retrying in #{delay}s..."
|
|
202
|
+
sleep delay
|
|
203
|
+
next
|
|
204
|
+
end
|
|
205
|
+
@file_system.warning error_message
|
|
206
|
+
raise "GitHub CLI command failed for #{@repo}: #{stderr}"
|
|
207
|
+
end
|
|
112
208
|
|
|
113
|
-
|
|
209
|
+
result = JSON.parse(stdout)
|
|
210
|
+
if result.nil? || (result.is_a?(Array) && result.empty?)
|
|
211
|
+
@file_system.warning "No data was found in GitHub for #{@repo}. Is that what you expected?"
|
|
212
|
+
end
|
|
213
|
+
return result
|
|
214
|
+
end
|
|
114
215
|
end
|
|
115
216
|
end
|
|
@@ -32,17 +32,21 @@ class JiraGateway
|
|
|
32
32
|
|
|
33
33
|
retries = 0
|
|
34
34
|
loop do
|
|
35
|
+
started = monotonic_time
|
|
35
36
|
stdout, stderr, status = capture3(command, stdin_data: stdin_data)
|
|
37
|
+
@file_system.diagnostic format('Jira call took %.2fs', monotonic_time - started)
|
|
36
38
|
|
|
37
39
|
if status.success?
|
|
38
40
|
@file_system.log "Returned (stderr): #{stderr.inspect}" unless stderr == ''
|
|
39
41
|
raise 'no response from curl on stdout' if stdout == ''
|
|
42
|
+
|
|
40
43
|
return parse_response(command: command, result: stdout)
|
|
41
44
|
end
|
|
42
45
|
|
|
43
|
-
if RETRYABLE_EXIT_CODES.include?(status.exitstatus) && retries < MAX_RETRIES
|
|
46
|
+
if RETRYABLE_EXIT_CODES.include?(status.exitstatus) && retries < MAX_RETRIES && !stderr.include?('503')
|
|
44
47
|
retries += 1
|
|
45
|
-
@file_system.log "Transient network error (exit #{status.exitstatus}), retrying in
|
|
48
|
+
@file_system.log "Transient network error (exit #{status.exitstatus}), retrying in " \
|
|
49
|
+
"#{RETRY_DELAY_SECONDS}s (attempt #{retries}/#{MAX_RETRIES})..."
|
|
46
50
|
sleep_between_retries
|
|
47
51
|
next
|
|
48
52
|
end
|
|
@@ -53,6 +57,11 @@ class JiraGateway
|
|
|
53
57
|
if stderr.include?('401')
|
|
54
58
|
raise 'The request was not authorized. Verify that your authentication token hasn\'t expired'
|
|
55
59
|
end
|
|
60
|
+
if stderr.include?('503')
|
|
61
|
+
raise 'Jira returned 503 (Service Unavailable). This may be a temporary outage, or your ' \
|
|
62
|
+
'Jira account may have been deactivated due to inactivity. Check your Jira subscription ' \
|
|
63
|
+
'and try again later.'
|
|
64
|
+
end
|
|
56
65
|
raise "Failed call with exit status #{status.exitstatus}. " \
|
|
57
66
|
"See #{@file_system.logfile_name} for details"
|
|
58
67
|
end
|
|
@@ -68,6 +77,11 @@ class JiraGateway
|
|
|
68
77
|
sleep RETRY_DELAY_SECONDS
|
|
69
78
|
end
|
|
70
79
|
|
|
80
|
+
def monotonic_time
|
|
81
|
+
# In its own method so we can mock it out in tests
|
|
82
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
83
|
+
end
|
|
84
|
+
|
|
71
85
|
def call_url relative_url:
|
|
72
86
|
command = make_curl_command url: "#{@jira_url}#{relative_url}"
|
|
73
87
|
exec_and_parse_response command: command, stdin_data: nil
|
|
@@ -478,22 +478,28 @@ class ProjectConfig
|
|
|
478
478
|
|
|
479
479
|
issues_path = File.join @target_path, "#{get_file_prefix}_issues"
|
|
480
480
|
if File.exist?(issues_path) && File.directory?(issues_path)
|
|
481
|
+
file_system.diagnostic "Loading issues from #{issues_path}"
|
|
481
482
|
issues = load_issues_from_issues_directory path: issues_path, timezone_offset: timezone_offset
|
|
483
|
+
file_system.diagnostic "Loaded #{issues.size} issues from disk"
|
|
482
484
|
else
|
|
483
485
|
file_system.log "Can't find directory #{issues_path}. Has a download been done?", also_write_to_stderr: true
|
|
484
486
|
return IssueCollection.new
|
|
485
487
|
end
|
|
486
488
|
|
|
487
489
|
# Attach related issues
|
|
490
|
+
file_system.diagnostic 'Starting attach phase'
|
|
491
|
+
issues_by_key = issues.to_h { |i| [i.key, i] }
|
|
488
492
|
issues.each do |i|
|
|
489
|
-
attach_subtasks issue: i,
|
|
490
|
-
attach_parent issue: i,
|
|
491
|
-
attach_linked_issues issue: i,
|
|
493
|
+
attach_subtasks issue: i, issues_by_key: issues_by_key
|
|
494
|
+
attach_parent issue: i, issues_by_key: issues_by_key
|
|
495
|
+
attach_linked_issues issue: i, issues_by_key: issues_by_key
|
|
492
496
|
end
|
|
497
|
+
file_system.diagnostic 'Attach phase complete'
|
|
493
498
|
|
|
494
499
|
# We'll have some issues that are in the list that weren't part of the initial query. Once we've
|
|
495
500
|
# attached them in the appropriate places, remove any that aren't part of that initial set.
|
|
496
501
|
issues.reject! { |i| !i.in_initial_query? } # rubocop:disable Style/InverseMethods
|
|
502
|
+
file_system.diagnostic "Retained #{issues.size} primary issues"
|
|
497
503
|
@issues = issues
|
|
498
504
|
attach_github_prs
|
|
499
505
|
end
|
|
@@ -501,24 +507,22 @@ class ProjectConfig
|
|
|
501
507
|
@issues
|
|
502
508
|
end
|
|
503
509
|
|
|
504
|
-
def attach_subtasks issue:,
|
|
510
|
+
def attach_subtasks issue:, issues_by_key:
|
|
505
511
|
issue.raw['fields']['subtasks']&.each do |subtask_element|
|
|
506
|
-
|
|
507
|
-
subtask = all_issues.find { |i| i.key == subtask_key }
|
|
512
|
+
subtask = issues_by_key[subtask_element['key']]
|
|
508
513
|
issue.subtasks << subtask if subtask
|
|
509
514
|
end
|
|
510
515
|
end
|
|
511
516
|
|
|
512
|
-
def attach_parent issue:,
|
|
517
|
+
def attach_parent issue:, issues_by_key:
|
|
513
518
|
parent_key = issue.parent_key
|
|
514
|
-
parent =
|
|
515
|
-
issue.parent = parent if parent
|
|
519
|
+
issue.parent = issues_by_key[parent_key] if parent_key
|
|
516
520
|
end
|
|
517
521
|
|
|
518
|
-
def attach_linked_issues issue:,
|
|
522
|
+
def attach_linked_issues issue:, issues_by_key:
|
|
519
523
|
issue.issue_links.each do |link|
|
|
520
524
|
if link.other_issue.artificial?
|
|
521
|
-
other =
|
|
525
|
+
other = issues_by_key[link.other_issue.key]
|
|
522
526
|
link.other_issue = other if other
|
|
523
527
|
end
|
|
524
528
|
end
|
|
@@ -638,6 +642,7 @@ class ProjectConfig
|
|
|
638
642
|
end
|
|
639
643
|
end
|
|
640
644
|
|
|
645
|
+
file_system.diagnostic "discard_changes_before: processing #{issues.size} issues"
|
|
641
646
|
issues.each do |issue|
|
|
642
647
|
cutoff_time = block.call(issue)
|
|
643
648
|
next if cutoff_time.nil?
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: jirametrics
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: '2.31'
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Bowler
|
|
@@ -210,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
210
210
|
- !ruby/object:Gem::Version
|
|
211
211
|
version: '0'
|
|
212
212
|
requirements: []
|
|
213
|
-
rubygems_version: 4.0.
|
|
213
|
+
rubygems_version: 4.0.13
|
|
214
214
|
specification_version: 4
|
|
215
215
|
summary: Extract Jira metrics
|
|
216
216
|
test_files: []
|