@rawdash/connector-github 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -51
- package/dist/index.d.ts +452 -2
- package/dist/index.js +455 -78
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,21 +1,62 @@
|
|
|
1
1
|
// ../../connector-shared/dist/index.js
|
|
2
2
|
var HTTP_CLIENT_VERSION = "0.0.0";
|
|
3
3
|
var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
function connectorUserAgent(connectorId) {
|
|
5
|
+
return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
|
|
6
|
+
}
|
|
7
|
+
function standardRateLimitPolicy(config) {
|
|
8
|
+
const { remainingHeader, resetHeader, resetUnit, resetFallbackMs } = config;
|
|
9
|
+
const multiplier = resetUnit === "s" ? 1e3 : 1;
|
|
10
|
+
return {
|
|
11
|
+
parse(h) {
|
|
12
|
+
const remainingRaw = h.get(remainingHeader);
|
|
13
|
+
if (remainingRaw === null || remainingRaw.trim() === "") {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
const remaining = Number(remainingRaw);
|
|
17
|
+
if (!Number.isFinite(remaining)) {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
const resetRaw = h.get(resetHeader);
|
|
21
|
+
if (resetRaw === null) {
|
|
22
|
+
if (resetFallbackMs === void 0) {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
remaining,
|
|
27
|
+
resetAt: new Date(Date.now() + resetFallbackMs)
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
if (resetRaw.trim() === "") {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
const reset = Number(resetRaw);
|
|
34
|
+
if (!Number.isFinite(reset) || reset < 0) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
const resetMs = reset * multiplier;
|
|
38
|
+
if (!Number.isFinite(resetMs)) {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
return { remaining, resetAt: new Date(resetMs) };
|
|
10
42
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function sanitizeAllowedUrl(options) {
|
|
46
|
+
const { url, host, pathname, protocol = "https:" } = options;
|
|
47
|
+
if (url === null) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
const u = new URL(url);
|
|
52
|
+
if (u.protocol !== protocol || u.host !== host || u.pathname !== pathname) {
|
|
14
53
|
return null;
|
|
15
54
|
}
|
|
16
|
-
return
|
|
55
|
+
return u.toString();
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
17
58
|
}
|
|
18
|
-
}
|
|
59
|
+
}
|
|
19
60
|
function parseLinkHeader(header) {
|
|
20
61
|
if (!header) {
|
|
21
62
|
return {};
|
|
@@ -34,7 +75,11 @@ function parseLinkHeader(header) {
|
|
|
34
75
|
import {
|
|
35
76
|
BaseConnector,
|
|
36
77
|
defineConfigFields,
|
|
37
|
-
|
|
78
|
+
defineConnectorDoc,
|
|
79
|
+
defineResources,
|
|
80
|
+
makeChunkedCursorGuard,
|
|
81
|
+
paginateChunked,
|
|
82
|
+
schemasFromResources
|
|
38
83
|
} from "@rawdash/core";
|
|
39
84
|
import { z } from "zod";
|
|
40
85
|
var configFields = defineConfigFields(
|
|
@@ -56,12 +101,41 @@ var configFields = defineConfigFields(
|
|
|
56
101
|
})
|
|
57
102
|
})
|
|
58
103
|
);
|
|
104
|
+
var doc = defineConnectorDoc({
|
|
105
|
+
displayName: "GitHub",
|
|
106
|
+
category: "engineering",
|
|
107
|
+
brandColor: "#181717",
|
|
108
|
+
tagline: "Sync pull requests, issues, deployments, releases, CI runs, and contributor activity from a GitHub repository.",
|
|
109
|
+
vendor: {
|
|
110
|
+
name: "GitHub",
|
|
111
|
+
apiDocs: "https://docs.github.com/rest",
|
|
112
|
+
website: "https://github.com"
|
|
113
|
+
},
|
|
114
|
+
auth: {
|
|
115
|
+
summary: "A personal access token is optional for public repositories but required for private repos and to avoid the low unauthenticated rate limit.",
|
|
116
|
+
setup: [
|
|
117
|
+
"Open GitHub \u2192 Settings \u2192 Developer settings \u2192 Personal access tokens.",
|
|
118
|
+
"Generate a token with the `repo` scope (read access is sufficient).",
|
|
119
|
+
'Store it as a secret and reference it from the connector config as `token: secret("GITHUB_TOKEN")`.'
|
|
120
|
+
]
|
|
121
|
+
},
|
|
122
|
+
rateLimit: "Unauthenticated requests share GitHub\u2019s low 60 requests/hour limit; an authenticated token raises it to 5,000 requests/hour.",
|
|
123
|
+
limitations: [
|
|
124
|
+
"The GitHub REST API can return the same item more than once within a sync (cursor pagination overlapping a mutating collection, retried requests, or an item surfaced via multiple endpoints). Each resource dedupes by stable id before writing, keeping the last copy seen.",
|
|
125
|
+
"Public repositories without a token are subject to GitHub\u2019s low unauthenticated rate limit."
|
|
126
|
+
]
|
|
127
|
+
});
|
|
59
128
|
var githubCredentials = {
|
|
60
129
|
token: {
|
|
61
130
|
description: "GitHub personal access token",
|
|
62
131
|
auth: "optional"
|
|
63
132
|
}
|
|
64
133
|
};
|
|
134
|
+
var githubRateLimit = standardRateLimitPolicy({
|
|
135
|
+
remainingHeader: "x-ratelimit-remaining",
|
|
136
|
+
resetHeader: "x-ratelimit-reset",
|
|
137
|
+
resetUnit: "s"
|
|
138
|
+
});
|
|
65
139
|
var PHASE_ORDER = [
|
|
66
140
|
"repo_stats",
|
|
67
141
|
"workflow_runs",
|
|
@@ -71,6 +145,23 @@ var PHASE_ORDER = [
|
|
|
71
145
|
"releases",
|
|
72
146
|
"contributors"
|
|
73
147
|
];
|
|
148
|
+
var PHASE_RESOURCES = {
|
|
149
|
+
repo_stats: ["repo"],
|
|
150
|
+
workflow_runs: ["workflow_run"],
|
|
151
|
+
pull_requests: ["pull_request"],
|
|
152
|
+
issues: ["issue"],
|
|
153
|
+
deployments: ["deployment"],
|
|
154
|
+
releases: ["release"],
|
|
155
|
+
contributors: ["contributor"]
|
|
156
|
+
};
|
|
157
|
+
function selectPhases(allowlist) {
|
|
158
|
+
if (allowlist === void 0) {
|
|
159
|
+
return PHASE_ORDER;
|
|
160
|
+
}
|
|
161
|
+
return PHASE_ORDER.filter(
|
|
162
|
+
(phase) => PHASE_RESOURCES[phase].some((r) => allowlist.has(r))
|
|
163
|
+
);
|
|
164
|
+
}
|
|
74
165
|
var CONTRIBUTORS_SKIPPED = /* @__PURE__ */ Symbol("contributors-skipped");
|
|
75
166
|
function dedupeByKey(items, keyFn, resource) {
|
|
76
167
|
if (items.length < 2) {
|
|
@@ -92,24 +183,256 @@ function dedupeByKey(items, keyFn, resource) {
|
|
|
92
183
|
}
|
|
93
184
|
return Array.from(seen.values());
|
|
94
185
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
186
|
+
var isGitHubSyncCursor = makeChunkedCursorGuard(PHASE_ORDER);
|
|
187
|
+
var workflowRunsResponseSchema = z.object({
|
|
188
|
+
total_count: z.number().int().optional(),
|
|
189
|
+
workflow_runs: z.array(
|
|
190
|
+
z.object({
|
|
191
|
+
id: z.number().int(),
|
|
192
|
+
name: z.string(),
|
|
193
|
+
conclusion: z.string().nullable(),
|
|
194
|
+
status: z.string(),
|
|
195
|
+
head_branch: z.string().nullable(),
|
|
196
|
+
actor: z.object({ login: z.string().min(1) }).nullable(),
|
|
197
|
+
created_at: z.iso.datetime(),
|
|
198
|
+
updated_at: z.iso.datetime(),
|
|
199
|
+
run_attempt: z.number().int(),
|
|
200
|
+
artifacts_url: z.string().optional(),
|
|
201
|
+
cancel_url: z.string().optional(),
|
|
202
|
+
check_suite_id: z.number().int().optional(),
|
|
203
|
+
check_suite_node_id: z.string().optional(),
|
|
204
|
+
check_suite_url: z.string().optional(),
|
|
205
|
+
display_title: z.string().optional(),
|
|
206
|
+
event: z.string().optional(),
|
|
207
|
+
head_commit: z.unknown().optional(),
|
|
208
|
+
head_repository: z.unknown().optional(),
|
|
209
|
+
head_sha: z.string().optional(),
|
|
210
|
+
html_url: z.string().optional(),
|
|
211
|
+
jobs_url: z.string().optional(),
|
|
212
|
+
logs_url: z.string().optional(),
|
|
213
|
+
node_id: z.string().optional(),
|
|
214
|
+
path: z.string().optional(),
|
|
215
|
+
previous_attempt_url: z.string().nullable().optional(),
|
|
216
|
+
pull_requests: z.array(z.unknown()).optional(),
|
|
217
|
+
referenced_workflows: z.array(z.unknown()).optional(),
|
|
218
|
+
repository: z.unknown().optional(),
|
|
219
|
+
rerun_url: z.string().optional(),
|
|
220
|
+
run_number: z.number().int().optional(),
|
|
221
|
+
run_started_at: z.iso.datetime().optional(),
|
|
222
|
+
triggering_actor: z.object({ login: z.string().min(1) }).optional(),
|
|
223
|
+
url: z.string().optional(),
|
|
224
|
+
workflow_id: z.number().int().optional(),
|
|
225
|
+
workflow_url: z.string().optional()
|
|
226
|
+
})
|
|
227
|
+
)
|
|
228
|
+
});
|
|
229
|
+
var pullRequestsSchema = z.array(
|
|
230
|
+
z.object({
|
|
231
|
+
number: z.number().int(),
|
|
232
|
+
title: z.string(),
|
|
233
|
+
state: z.string(),
|
|
234
|
+
draft: z.boolean(),
|
|
235
|
+
user: z.object({
|
|
236
|
+
login: z.string().min(1),
|
|
237
|
+
avatar_url: z.string().optional(),
|
|
238
|
+
events_url: z.string().optional(),
|
|
239
|
+
followers_url: z.string().optional(),
|
|
240
|
+
following_url: z.string().optional(),
|
|
241
|
+
gists_url: z.string().optional(),
|
|
242
|
+
gravatar_id: z.string().nullable().optional(),
|
|
243
|
+
html_url: z.string().optional(),
|
|
244
|
+
id: z.number().int().optional(),
|
|
245
|
+
node_id: z.string().optional(),
|
|
246
|
+
organizations_url: z.string().optional(),
|
|
247
|
+
received_events_url: z.string().optional(),
|
|
248
|
+
repos_url: z.string().optional(),
|
|
249
|
+
site_admin: z.boolean().optional(),
|
|
250
|
+
starred_url: z.string().optional(),
|
|
251
|
+
subscriptions_url: z.string().optional(),
|
|
252
|
+
type: z.string().optional(),
|
|
253
|
+
url: z.string().optional(),
|
|
254
|
+
user_view_type: z.string().optional()
|
|
255
|
+
}),
|
|
256
|
+
created_at: z.iso.datetime(),
|
|
257
|
+
updated_at: z.iso.datetime(),
|
|
258
|
+
_links: z.unknown().optional(),
|
|
259
|
+
active_lock_reason: z.string().nullable().optional(),
|
|
260
|
+
assignee: z.unknown().optional(),
|
|
261
|
+
assignees: z.unknown().optional(),
|
|
262
|
+
author_association: z.string().optional(),
|
|
263
|
+
auto_merge: z.unknown().optional(),
|
|
264
|
+
base: z.unknown().optional(),
|
|
265
|
+
body: z.string().nullable().optional(),
|
|
266
|
+
closed_at: z.string().nullable().optional(),
|
|
267
|
+
comments_url: z.string().optional(),
|
|
268
|
+
commits_url: z.string().optional(),
|
|
269
|
+
diff_url: z.string().optional(),
|
|
270
|
+
head: z.unknown().optional(),
|
|
271
|
+
html_url: z.string().optional(),
|
|
272
|
+
id: z.number().int().optional(),
|
|
273
|
+
issue_url: z.string().optional(),
|
|
274
|
+
labels: z.unknown().optional(),
|
|
275
|
+
locked: z.boolean().optional(),
|
|
276
|
+
merge_commit_sha: z.string().nullable().optional(),
|
|
277
|
+
merged_at: z.string().nullable().optional(),
|
|
278
|
+
milestone: z.unknown().optional(),
|
|
279
|
+
node_id: z.string().optional(),
|
|
280
|
+
patch_url: z.string().optional(),
|
|
281
|
+
requested_reviewers: z.unknown().optional(),
|
|
282
|
+
requested_teams: z.unknown().optional(),
|
|
283
|
+
review_comment_url: z.string().optional(),
|
|
284
|
+
review_comments_url: z.string().optional(),
|
|
285
|
+
statuses_url: z.string().optional(),
|
|
286
|
+
url: z.string().optional()
|
|
287
|
+
})
|
|
288
|
+
);
|
|
289
|
+
var reviewsSchema = z.array(
|
|
290
|
+
z.object({
|
|
291
|
+
user: z.object({ login: z.string().min(1) }).nullable(),
|
|
292
|
+
state: z.string(),
|
|
293
|
+
submitted_at: z.iso.datetime()
|
|
294
|
+
})
|
|
295
|
+
);
|
|
296
|
+
var issuesSchema = z.array(
|
|
297
|
+
z.object({
|
|
298
|
+
number: z.number().int(),
|
|
299
|
+
title: z.string(),
|
|
300
|
+
state: z.string(),
|
|
301
|
+
labels: z.array(z.object({ name: z.string() })),
|
|
302
|
+
assignees: z.array(z.object({ login: z.string().min(1) })),
|
|
303
|
+
user: z.object({ login: z.string().min(1) }).catchall(z.unknown()),
|
|
304
|
+
created_at: z.iso.datetime(),
|
|
305
|
+
updated_at: z.iso.datetime(),
|
|
306
|
+
closed_at: z.iso.datetime().nullable(),
|
|
307
|
+
pull_request: z.unknown().optional(),
|
|
308
|
+
active_lock_reason: z.unknown().optional(),
|
|
309
|
+
assignee: z.unknown().optional(),
|
|
310
|
+
author_association: z.string().optional(),
|
|
311
|
+
body: z.string().nullable().optional(),
|
|
312
|
+
closed_by: z.unknown().optional(),
|
|
313
|
+
comments: z.number().int().optional(),
|
|
314
|
+
comments_url: z.string().optional(),
|
|
315
|
+
draft: z.boolean().optional(),
|
|
316
|
+
events_url: z.string().optional(),
|
|
317
|
+
html_url: z.string().optional(),
|
|
318
|
+
id: z.number().int().optional(),
|
|
319
|
+
issue_field_values: z.unknown().optional(),
|
|
320
|
+
labels_url: z.string().optional(),
|
|
321
|
+
locked: z.boolean().optional(),
|
|
322
|
+
milestone: z.unknown().optional(),
|
|
323
|
+
node_id: z.string().optional(),
|
|
324
|
+
performed_via_github_app: z.unknown().optional(),
|
|
325
|
+
reactions: z.unknown().optional(),
|
|
326
|
+
repository_url: z.string().optional(),
|
|
327
|
+
state_reason: z.unknown().optional(),
|
|
328
|
+
timeline_url: z.string().optional(),
|
|
329
|
+
type: z.unknown().optional(),
|
|
330
|
+
url: z.string().optional()
|
|
331
|
+
})
|
|
332
|
+
);
|
|
333
|
+
var deploymentsSchema = z.array(
|
|
334
|
+
z.object({
|
|
335
|
+
id: z.number().int(),
|
|
336
|
+
environment: z.string(),
|
|
337
|
+
ref: z.string(),
|
|
338
|
+
sha: z.string(),
|
|
339
|
+
creator: z.object({ login: z.string().min(1) }).nullable(),
|
|
340
|
+
created_at: z.iso.datetime()
|
|
341
|
+
})
|
|
342
|
+
);
|
|
343
|
+
var deploymentStatusesSchema = z.array(
|
|
344
|
+
z.object({
|
|
345
|
+
state: z.string(),
|
|
346
|
+
updated_at: z.iso.datetime()
|
|
347
|
+
})
|
|
348
|
+
);
|
|
349
|
+
var releasesSchema = z.array(
|
|
350
|
+
z.object({
|
|
351
|
+
id: z.number().int(),
|
|
352
|
+
tag_name: z.string(),
|
|
353
|
+
name: z.string().nullable(),
|
|
354
|
+
draft: z.boolean(),
|
|
355
|
+
prerelease: z.boolean(),
|
|
356
|
+
created_at: z.iso.datetime(),
|
|
357
|
+
published_at: z.iso.datetime().nullable(),
|
|
358
|
+
author: z.object({ login: z.string().min(1) })
|
|
359
|
+
})
|
|
360
|
+
);
|
|
361
|
+
var contributorsSchema = z.array(
|
|
362
|
+
z.object({
|
|
363
|
+
total: z.number().int(),
|
|
364
|
+
weeks: z.array(
|
|
365
|
+
z.object({
|
|
366
|
+
w: z.number().int(),
|
|
367
|
+
a: z.number().int(),
|
|
368
|
+
d: z.number().int(),
|
|
369
|
+
c: z.number().int()
|
|
370
|
+
})
|
|
371
|
+
),
|
|
372
|
+
author: z.object({ login: z.string().min(1) })
|
|
373
|
+
})
|
|
374
|
+
);
|
|
375
|
+
var repoStatsSchema = z.object({
|
|
376
|
+
stargazers_count: z.number().int(),
|
|
377
|
+
forks_count: z.number().int(),
|
|
378
|
+
subscribers_count: z.number().int()
|
|
379
|
+
});
|
|
380
|
+
var githubResources = defineResources({
|
|
381
|
+
repo: {
|
|
382
|
+
shape: "entity",
|
|
383
|
+
description: "Top-level repository stats (stars, forks, and watchers) as a single entity.",
|
|
384
|
+
endpoint: "GET /repos/{owner}/{repo}",
|
|
385
|
+
responses: { repo: repoStatsSchema }
|
|
386
|
+
},
|
|
387
|
+
workflow_run: {
|
|
388
|
+
shape: "event",
|
|
389
|
+
description: "GitHub Actions CI pipeline executions.",
|
|
390
|
+
endpoint: "GET /repos/{owner}/{repo}/actions/runs",
|
|
391
|
+
responses: { workflow_runs: workflowRunsResponseSchema }
|
|
392
|
+
},
|
|
393
|
+
pull_request: {
|
|
394
|
+
shape: "entity",
|
|
395
|
+
description: "Open and closed pull requests, including draft state, author, and review state.",
|
|
396
|
+
endpoint: "GET /repos/{owner}/{repo}/pulls",
|
|
397
|
+
notes: "Review state is folded in from GET /repos/{owner}/{repo}/pulls/{number}/reviews per PR.",
|
|
398
|
+
responses: {
|
|
399
|
+
pull_requests: pullRequestsSchema,
|
|
400
|
+
pull_request_reviews: reviewsSchema
|
|
401
|
+
}
|
|
402
|
+
},
|
|
403
|
+
issue: {
|
|
404
|
+
shape: "entity",
|
|
405
|
+
description: "Open and closed issues with labels, assignees, and author (pull requests excluded).",
|
|
406
|
+
endpoint: "GET /repos/{owner}/{repo}/issues",
|
|
407
|
+
responses: { issues: issuesSchema }
|
|
408
|
+
},
|
|
409
|
+
deployment: {
|
|
410
|
+
shape: "entity",
|
|
411
|
+
description: "Deployments with their latest status, keyed by environment and ref.",
|
|
412
|
+
endpoint: "GET /repos/{owner}/{repo}/deployments",
|
|
413
|
+
notes: "The latest status is folded in from GET /repos/{owner}/{repo}/deployments/{id}/statuses.",
|
|
414
|
+
responses: {
|
|
415
|
+
deployments: deploymentsSchema,
|
|
416
|
+
deployment_statuses: deploymentStatusesSchema
|
|
417
|
+
}
|
|
418
|
+
},
|
|
419
|
+
release: {
|
|
420
|
+
shape: "entity",
|
|
421
|
+
description: "Published, draft, and prerelease GitHub releases.",
|
|
422
|
+
endpoint: "GET /repos/{owner}/{repo}/releases",
|
|
423
|
+
responses: { releases: releasesSchema }
|
|
424
|
+
},
|
|
425
|
+
contributor: {
|
|
426
|
+
shape: "entity",
|
|
427
|
+
description: "Per-author commit activity (commits, additions, deletions) for the repository.",
|
|
428
|
+
endpoint: "GET /repos/{owner}/{repo}/stats/contributors",
|
|
429
|
+
responses: { contributors: contributorsSchema }
|
|
430
|
+
}
|
|
431
|
+
});
|
|
111
432
|
var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
112
433
|
static id = "github-actions";
|
|
434
|
+
static resources = githubResources;
|
|
435
|
+
static schemas = schemasFromResources(githubResources);
|
|
113
436
|
static create(input, ctx) {
|
|
114
437
|
const parsed = configFields.parse(input);
|
|
115
438
|
return new _GitHubConnector(
|
|
@@ -121,11 +444,12 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
121
444
|
id = "github-actions";
|
|
122
445
|
credentials = githubCredentials;
|
|
123
446
|
seenWorkflowRunIds = /* @__PURE__ */ new Set();
|
|
447
|
+
preservedDeploymentStatus = /* @__PURE__ */ new Map();
|
|
124
448
|
buildHeaders() {
|
|
125
449
|
const headers = {
|
|
126
450
|
Accept: "application/vnd.github+json",
|
|
127
451
|
"X-GitHub-Api-Version": "2022-11-28",
|
|
128
|
-
"User-Agent": "
|
|
452
|
+
"User-Agent": connectorUserAgent("github")
|
|
129
453
|
};
|
|
130
454
|
if (this.creds.token) {
|
|
131
455
|
headers["Authorization"] = `Bearer ${this.creds.token}`;
|
|
@@ -159,22 +483,21 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
159
483
|
}
|
|
160
484
|
}
|
|
161
485
|
sanitizePageUrl(phase, pageUrl) {
|
|
162
|
-
if (pageUrl === null) {
|
|
163
|
-
return null;
|
|
164
|
-
}
|
|
165
486
|
const allowedPath = this.allowedPageBasePath(phase);
|
|
166
487
|
if (allowedPath === null) {
|
|
167
488
|
return null;
|
|
168
489
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
490
|
+
return sanitizeAllowedUrl({
|
|
491
|
+
url: pageUrl,
|
|
492
|
+
host: "api.github.com",
|
|
493
|
+
pathname: allowedPath
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
isResourceAllowed(options, resource) {
|
|
497
|
+
if (!options.resources) {
|
|
498
|
+
return true;
|
|
177
499
|
}
|
|
500
|
+
return options.resources.has(resource);
|
|
178
501
|
}
|
|
179
502
|
resolveCursor(cursor) {
|
|
180
503
|
if (!isGitHubSyncCursor(cursor)) {
|
|
@@ -230,24 +553,30 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
230
553
|
next: cutoffReached ? null : nextLink
|
|
231
554
|
};
|
|
232
555
|
}
|
|
233
|
-
async fetchPullRequests(page, signal) {
|
|
556
|
+
async fetchPullRequests(options, page, signal) {
|
|
234
557
|
const { owner, repo } = this.settings;
|
|
235
|
-
const url = page ?? `https://api.github.com/repos/${owner}/${repo}/pulls?state=all&per_page=100`;
|
|
558
|
+
const url = page ?? `https://api.github.com/repos/${owner}/${repo}/pulls?state=all&sort=updated&direction=desc&per_page=100`;
|
|
236
559
|
const res = await this.fetch(url, "pull_requests", signal);
|
|
237
560
|
const nextLink = parseLinkHeader(res.headers.get("link"))["next"] ?? null;
|
|
238
561
|
const prs = res.body;
|
|
562
|
+
const cutoff = options.since ? new Date(options.since).getTime() : null;
|
|
563
|
+
const filteredPrs = cutoff !== null ? prs.filter((pr) => new Date(pr.updated_at).getTime() >= cutoff) : prs;
|
|
564
|
+
const lastPr = prs.at(-1);
|
|
565
|
+
const cutoffReached = cutoff !== null && lastPr !== void 0 && new Date(lastPr.updated_at).getTime() < cutoff;
|
|
239
566
|
const reviewsByPR = /* @__PURE__ */ new Map();
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
567
|
+
if (this.isResourceAllowed(options, "pull_request_reviews")) {
|
|
568
|
+
for (const pr of filteredPrs) {
|
|
569
|
+
signal?.throwIfAborted();
|
|
570
|
+
const reviews = await this.fetch(
|
|
571
|
+
`https://api.github.com/repos/${owner}/${repo}/pulls/${pr.number}/reviews`,
|
|
572
|
+
"pull_request_reviews",
|
|
573
|
+
signal
|
|
574
|
+
);
|
|
575
|
+
reviewsByPR.set(pr.number, reviews.body);
|
|
576
|
+
}
|
|
248
577
|
}
|
|
249
|
-
const items = [{ prs, reviewsByPR }];
|
|
250
|
-
return { items, next: nextLink };
|
|
578
|
+
const items = [{ prs: filteredPrs, reviewsByPR }];
|
|
579
|
+
return { items, next: cutoffReached ? null : nextLink };
|
|
251
580
|
}
|
|
252
581
|
async fetchIssues(options, page, signal) {
|
|
253
582
|
const { owner, repo } = this.settings;
|
|
@@ -267,7 +596,7 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
267
596
|
const nextLink = parseLinkHeader(res.headers.get("link"))["next"] ?? null;
|
|
268
597
|
return { items: res.body, next: nextLink };
|
|
269
598
|
}
|
|
270
|
-
async fetchDeployments(page, signal) {
|
|
599
|
+
async fetchDeployments(options, page, signal) {
|
|
271
600
|
const { owner, repo } = this.settings;
|
|
272
601
|
const url = page ?? `https://api.github.com/repos/${owner}/${repo}/deployments?per_page=100`;
|
|
273
602
|
const res = await this.fetch(
|
|
@@ -277,25 +606,41 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
277
606
|
);
|
|
278
607
|
const nextLink = parseLinkHeader(res.headers.get("link"))["next"] ?? null;
|
|
279
608
|
const deployments = res.body;
|
|
609
|
+
const cutoff = options.since ? new Date(options.since).getTime() : null;
|
|
610
|
+
const filteredDeployments = cutoff !== null ? deployments.filter((d) => new Date(d.created_at).getTime() >= cutoff) : deployments;
|
|
611
|
+
const lastDeployment = deployments.at(-1);
|
|
612
|
+
const cutoffReached = cutoff !== null && lastDeployment !== void 0 && new Date(lastDeployment.created_at).getTime() < cutoff;
|
|
280
613
|
const latestStatusById = /* @__PURE__ */ new Map();
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
614
|
+
if (this.isResourceAllowed(options, "deployment_statuses")) {
|
|
615
|
+
for (const deployment of filteredDeployments) {
|
|
616
|
+
signal?.throwIfAborted();
|
|
617
|
+
const statusRes = await this.fetch(
|
|
618
|
+
`https://api.github.com/repos/${owner}/${repo}/deployments/${deployment.id}/statuses?per_page=1`,
|
|
619
|
+
"deployment_statuses",
|
|
620
|
+
signal
|
|
621
|
+
);
|
|
622
|
+
latestStatusById.set(deployment.id, statusRes.body[0] ?? null);
|
|
623
|
+
}
|
|
289
624
|
}
|
|
290
|
-
const items = [
|
|
291
|
-
|
|
625
|
+
const items = [
|
|
626
|
+
{ deployments: filteredDeployments, latestStatusById }
|
|
627
|
+
];
|
|
628
|
+
return { items, next: cutoffReached ? null : nextLink };
|
|
292
629
|
}
|
|
293
|
-
async fetchReleases(page, signal) {
|
|
630
|
+
async fetchReleases(options, page, signal) {
|
|
294
631
|
const { owner, repo } = this.settings;
|
|
295
632
|
const url = page ?? `https://api.github.com/repos/${owner}/${repo}/releases?per_page=100`;
|
|
296
633
|
const res = await this.fetch(url, "releases", signal);
|
|
297
634
|
const nextLink = parseLinkHeader(res.headers.get("link"))["next"] ?? null;
|
|
298
|
-
|
|
635
|
+
const releases = res.body;
|
|
636
|
+
const cutoff = options.since ? new Date(options.since).getTime() : null;
|
|
637
|
+
const filtered = cutoff !== null ? releases.filter((r) => {
|
|
638
|
+
const ts = new Date(r.published_at ?? r.created_at).getTime();
|
|
639
|
+
return ts >= cutoff;
|
|
640
|
+
}) : releases;
|
|
641
|
+
const lastRelease = releases.at(-1);
|
|
642
|
+
const cutoffReached = cutoff !== null && lastRelease !== void 0 && new Date(lastRelease.published_at ?? lastRelease.created_at).getTime() < cutoff;
|
|
643
|
+
return { items: filtered, next: cutoffReached ? null : nextLink };
|
|
299
644
|
}
|
|
300
645
|
async fetchContributors(signal) {
|
|
301
646
|
const { owner, repo } = this.settings;
|
|
@@ -409,10 +754,16 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
409
754
|
});
|
|
410
755
|
}
|
|
411
756
|
}
|
|
412
|
-
async writePullRequests(storage, items, page) {
|
|
757
|
+
async writePullRequests(storage, items, page, options) {
|
|
758
|
+
const reviewsAllowed = this.isResourceAllowed(
|
|
759
|
+
options,
|
|
760
|
+
"pull_request_reviews"
|
|
761
|
+
);
|
|
413
762
|
if (page === null) {
|
|
414
763
|
await storage.entities([], { types: ["pull_request"] });
|
|
415
|
-
|
|
764
|
+
if (reviewsAllowed) {
|
|
765
|
+
await storage.edges([], { kinds: ["reviewed_by"] });
|
|
766
|
+
}
|
|
416
767
|
}
|
|
417
768
|
const pageItems = items;
|
|
418
769
|
for (const { prs: rawPrs, reviewsByPR } of pageItems) {
|
|
@@ -435,6 +786,9 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
435
786
|
updated_at: new Date(pr.updated_at).getTime()
|
|
436
787
|
});
|
|
437
788
|
}
|
|
789
|
+
if (!reviewsAllowed) {
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
438
792
|
for (const pr of prs) {
|
|
439
793
|
const reviews = reviewsByPR.get(pr.number) ?? [];
|
|
440
794
|
for (const review of reviews) {
|
|
@@ -482,8 +836,21 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
482
836
|
});
|
|
483
837
|
}
|
|
484
838
|
}
|
|
485
|
-
async writeDeployments(storage, items, page) {
|
|
839
|
+
async writeDeployments(storage, items, page, options) {
|
|
840
|
+
const statusesAllowed = this.isResourceAllowed(
|
|
841
|
+
options,
|
|
842
|
+
"deployment_statuses"
|
|
843
|
+
);
|
|
486
844
|
if (page === null) {
|
|
845
|
+
if (!statusesAllowed) {
|
|
846
|
+
const existing = await storage.queryEntities({ type: "deployment" });
|
|
847
|
+
for (const entity of existing) {
|
|
848
|
+
const prev = entity.attributes["latest_status"];
|
|
849
|
+
if (typeof prev === "string") {
|
|
850
|
+
this.preservedDeploymentStatus.set(entity.id, prev);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
487
854
|
await storage.entities([], { types: ["deployment"] });
|
|
488
855
|
}
|
|
489
856
|
const pageItems = items;
|
|
@@ -494,9 +861,16 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
494
861
|
"deployments"
|
|
495
862
|
);
|
|
496
863
|
for (const deployment of deployments) {
|
|
497
|
-
const status = latestStatusById.get(deployment.id) ?? null;
|
|
498
864
|
const createdMs = new Date(deployment.created_at).getTime();
|
|
499
|
-
|
|
865
|
+
let latestStatus;
|
|
866
|
+
let statusUpdatedMs = null;
|
|
867
|
+
if (statusesAllowed) {
|
|
868
|
+
const status = latestStatusById.get(deployment.id) ?? null;
|
|
869
|
+
latestStatus = status?.state ?? "unknown";
|
|
870
|
+
statusUpdatedMs = status?.updated_at ? new Date(status.updated_at).getTime() : null;
|
|
871
|
+
} else {
|
|
872
|
+
latestStatus = this.preservedDeploymentStatus.get(String(deployment.id)) ?? "unknown";
|
|
873
|
+
}
|
|
500
874
|
await storage.entity({
|
|
501
875
|
type: "deployment",
|
|
502
876
|
id: String(deployment.id),
|
|
@@ -506,7 +880,7 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
506
880
|
sha: deployment.sha,
|
|
507
881
|
creator: deployment.creator?.login ?? "",
|
|
508
882
|
created_at: createdMs,
|
|
509
|
-
latest_status:
|
|
883
|
+
latest_status: latestStatus
|
|
510
884
|
},
|
|
511
885
|
updated_at: Math.max(createdMs, statusUpdatedMs ?? 0)
|
|
512
886
|
});
|
|
@@ -572,10 +946,12 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
572
946
|
}
|
|
573
947
|
async sync(options, storage, signal) {
|
|
574
948
|
const cursor = this.resolveCursor(options.cursor);
|
|
949
|
+
const phases = selectPhases(options.resources);
|
|
575
950
|
return paginateChunked({
|
|
576
|
-
phases
|
|
951
|
+
phases,
|
|
577
952
|
cursor,
|
|
578
953
|
signal,
|
|
954
|
+
logger: this.logger,
|
|
579
955
|
fetchPage: async (phase, page, sig) => {
|
|
580
956
|
switch (phase) {
|
|
581
957
|
case "repo_stats":
|
|
@@ -583,13 +959,13 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
583
959
|
case "workflow_runs":
|
|
584
960
|
return options.mode === "latest" ? this.fetchWorkflowRunsLatest(sig) : this.fetchWorkflowRunsFull(options, page, sig);
|
|
585
961
|
case "pull_requests":
|
|
586
|
-
return this.fetchPullRequests(page, sig);
|
|
962
|
+
return this.fetchPullRequests(options, page, sig);
|
|
587
963
|
case "issues":
|
|
588
964
|
return this.fetchIssues(options, page, sig);
|
|
589
965
|
case "deployments":
|
|
590
|
-
return this.fetchDeployments(page, sig);
|
|
966
|
+
return this.fetchDeployments(options, page, sig);
|
|
591
967
|
case "releases":
|
|
592
|
-
return this.fetchReleases(page, sig);
|
|
968
|
+
return this.fetchReleases(options, page, sig);
|
|
593
969
|
case "contributors":
|
|
594
970
|
return this.fetchContributors(sig);
|
|
595
971
|
}
|
|
@@ -601,11 +977,11 @@ var GitHubConnector = class _GitHubConnector extends BaseConnector {
|
|
|
601
977
|
case "workflow_runs":
|
|
602
978
|
return options.mode === "latest" ? this.writeWorkflowRunsLatest(storage, items) : this.writeWorkflowRunsFull(storage, items, page);
|
|
603
979
|
case "pull_requests":
|
|
604
|
-
return this.writePullRequests(storage, items, page);
|
|
980
|
+
return this.writePullRequests(storage, items, page, options);
|
|
605
981
|
case "issues":
|
|
606
982
|
return this.writeIssues(storage, items, page);
|
|
607
983
|
case "deployments":
|
|
608
|
-
return this.writeDeployments(storage, items, page);
|
|
984
|
+
return this.writeDeployments(storage, items, page, options);
|
|
609
985
|
case "releases":
|
|
610
986
|
return this.writeReleases(storage, items, page);
|
|
611
987
|
case "contributors":
|
|
@@ -621,6 +997,7 @@ var index_default = GitHubConnector;
|
|
|
621
997
|
export {
|
|
622
998
|
GitHubConnector,
|
|
623
999
|
configFields,
|
|
624
|
-
index_default as default
|
|
1000
|
+
index_default as default,
|
|
1001
|
+
doc
|
|
625
1002
|
};
|
|
626
1003
|
//# sourceMappingURL=index.js.map
|