@pepys/source-github 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -5
- package/src/consumers/github-pull-request.ts +67 -1
- package/src/families/issue-comment-family.ts +19 -0
- package/src/families/repository-family.ts +28 -0
- package/src/families/review-comment-family.ts +18 -0
- package/src/families/review-thread-family.ts +19 -0
- package/src/generated/github-pull-request-boundary.ts +51 -0
- package/src/generated/github-pull-request.sql +9 -0
- package/src/github-pull-request-consumer.ts +14 -1
- package/src/github-source.ts +9 -1
- package/src/ingest/build-github-fact.ts +1 -1
- package/src/ingest/github-app/read-installation-token.ts +4 -1
- package/src/ingest/github-app/request-github-api.ts +30 -9
- package/src/ingest/github-event-rules.ts +128 -0
- package/src/ingest/recovery/pull-failed-deliveries.ts +10 -6
- package/src/ingest/resync/pull-marked-objects.ts +12 -6
- /package/src/{utils → ingest}/build-body-hash.ts +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pepys/source-github",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "The pepys GitHub plug-in: pull requests, releases, commits and deploys",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -19,9 +19,9 @@
|
|
|
19
19
|
"access": "public"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@pepys/api": "0.
|
|
23
|
-
"@pepys/ingest": "0.
|
|
24
|
-
"@pepys/kit": "0.
|
|
22
|
+
"@pepys/api": "0.4.1",
|
|
23
|
+
"@pepys/ingest": "0.4.1",
|
|
24
|
+
"@pepys/kit": "0.4.1",
|
|
25
25
|
"@zgeoff/std": "0.4.3",
|
|
26
26
|
"kysely": "0.29.8",
|
|
27
27
|
"ts-pattern": "5.9.2",
|
|
@@ -30,7 +30,8 @@
|
|
|
30
30
|
"devDependencies": {
|
|
31
31
|
"@msw/data": "1.1.8",
|
|
32
32
|
"@orpc/client": "2.0.0-beta.50",
|
|
33
|
-
"@pepys/
|
|
33
|
+
"@pepys/mock-network": "0.0.0",
|
|
34
|
+
"@pepys/testing": "0.4.1",
|
|
34
35
|
"msw": "3.0.3",
|
|
35
36
|
"tiny-invariant": "1.3.3"
|
|
36
37
|
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
initGithubPullRequestModel,
|
|
3
|
+
readGithubPullRequestAliasRow,
|
|
3
4
|
readGithubPullRequestRow,
|
|
4
5
|
toGithubPullRequestID,
|
|
5
6
|
updateGithubPullRequestModel,
|
|
6
7
|
} from '../generated/github-pull-request-boundary.ts';
|
|
7
8
|
import type {
|
|
8
9
|
EntityID,
|
|
10
|
+
GithubPullRequestAliasRow,
|
|
9
11
|
GithubPullRequestMessage,
|
|
10
12
|
GithubPullRequestModel,
|
|
11
13
|
GithubPullRequestOperation,
|
|
@@ -64,7 +66,71 @@ function mapFact(factJSON: string): GithubPullRequestOperation[] {
|
|
|
64
66
|
.filter((bump) => !replacedKeys.has(JSON.stringify(bump)))
|
|
65
67
|
.map((bump) => ({ upsert: { github_release_bump: bump } }));
|
|
66
68
|
|
|
67
|
-
|
|
69
|
+
const aliases: GithubPullRequestOperation[] = collectAliases(row)
|
|
70
|
+
.filter((alias) => shouldTakeAlias(alias, readGithubPullRequestAliasRow(alias.alias)))
|
|
71
|
+
.map((alias) => ({ upsert: { github_pull_request_alias: alias } }));
|
|
72
|
+
|
|
73
|
+
return [{ upsert: { github_pull_request: row } }, ...aliases, ...removed, ...added];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// The URL, and `<owner>/<repo>#<number>` when the URL has the `/<owner>/<repo>/pull/<number>`
|
|
77
|
+
// path. GitHub matches an owner and a repository in any case, so each alias is in lowercase.
|
|
78
|
+
function collectAliases(row: GithubPullRequestRow): GithubPullRequestAliasRow[] {
|
|
79
|
+
const url = trimTrailingSlash(row.url.toLowerCase());
|
|
80
|
+
|
|
81
|
+
const aliases: GithubPullRequestAliasRow[] = [
|
|
82
|
+
{ alias: url, entity_id: row.id, updated_at: row.updated_at },
|
|
83
|
+
];
|
|
84
|
+
|
|
85
|
+
const parts = url.split('/');
|
|
86
|
+
const [scheme, gap, host, owner, repository, pull, number] = parts;
|
|
87
|
+
|
|
88
|
+
if (
|
|
89
|
+
parts.length !== 7 ||
|
|
90
|
+
scheme === undefined ||
|
|
91
|
+
!scheme.endsWith(':') ||
|
|
92
|
+
gap !== '' ||
|
|
93
|
+
host === undefined ||
|
|
94
|
+
host === '' ||
|
|
95
|
+
owner === undefined ||
|
|
96
|
+
owner === '' ||
|
|
97
|
+
repository === undefined ||
|
|
98
|
+
repository === '' ||
|
|
99
|
+
pull !== 'pull' ||
|
|
100
|
+
number !== `${row.number}`
|
|
101
|
+
) {
|
|
102
|
+
return aliases;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
aliases.push({
|
|
106
|
+
alias: `${owner}/${repository}#${row.number}`,
|
|
107
|
+
entity_id: row.id,
|
|
108
|
+
updated_at: row.updated_at,
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
return aliases;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function trimTrailingSlash(text: string): string {
|
|
115
|
+
return text.endsWith('/') ? text.slice(0, -1) : text;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// The holder's own row is no guide after a rename, since its newest fact may name another
|
|
119
|
+
// repository. The alias time keeps the outcome free of arrival order, and a tie between two pull
|
|
120
|
+
// requests goes to the later fact, as it does for the pull request row.
|
|
121
|
+
function shouldTakeAlias(
|
|
122
|
+
alias: GithubPullRequestAliasRow,
|
|
123
|
+
holding: GithubPullRequestAliasRow | undefined,
|
|
124
|
+
): boolean {
|
|
125
|
+
if (holding === undefined) {
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (holding.updated_at !== alias.updated_at) {
|
|
130
|
+
return holding.updated_at < alias.updated_at;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return holding.entity_id !== alias.entity_id;
|
|
68
134
|
}
|
|
69
135
|
|
|
70
136
|
const releaseURLStart = 'https://github.com/';
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { defineFactFamily } from '@pepys/kit';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { buildFamilyTypes } from './build-family-types.ts';
|
|
4
|
+
|
|
5
|
+
// The `action` values of GitHub's `issue_comment` webhook event, which covers the conversation
|
|
6
|
+
// comments of both issues and pull requests.
|
|
7
|
+
const actions = ['created', 'deleted', 'edited', 'pinned', 'unpinned'] as const;
|
|
8
|
+
|
|
9
|
+
const payloadSchema = z.looseObject({
|
|
10
|
+
action: z.string(),
|
|
11
|
+
comment: z.looseObject({ id: z.number().int() }),
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
export const issueCommentFamily = defineFactFamily({
|
|
15
|
+
stream: 'facts.github.issue_comment',
|
|
16
|
+
api: 'github/2022-11-28',
|
|
17
|
+
entityKind: 'github:issue_comment',
|
|
18
|
+
types: buildFamilyTypes('issue_comment', actions, payloadSchema),
|
|
19
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { defineFactFamily } from '@pepys/kit';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { buildFamilyTypes } from './build-family-types.ts';
|
|
4
|
+
|
|
5
|
+
// The `action` values of GitHub's `repository` webhook event.
|
|
6
|
+
const actions = [
|
|
7
|
+
'archived',
|
|
8
|
+
'created',
|
|
9
|
+
'deleted',
|
|
10
|
+
'edited',
|
|
11
|
+
'privatized',
|
|
12
|
+
'publicized',
|
|
13
|
+
'renamed',
|
|
14
|
+
'transferred',
|
|
15
|
+
'unarchived',
|
|
16
|
+
] as const;
|
|
17
|
+
|
|
18
|
+
const payloadSchema = z.looseObject({
|
|
19
|
+
action: z.string(),
|
|
20
|
+
repository: z.looseObject({ id: z.number().int() }),
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
export const repositoryFamily = defineFactFamily({
|
|
24
|
+
stream: 'facts.github.repository',
|
|
25
|
+
api: 'github/2022-11-28',
|
|
26
|
+
entityKind: 'github:repo',
|
|
27
|
+
types: buildFamilyTypes('repository', actions, payloadSchema),
|
|
28
|
+
});
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { defineFactFamily } from '@pepys/kit';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { buildFamilyTypes } from './build-family-types.ts';
|
|
4
|
+
|
|
5
|
+
// The `action` values of GitHub's `pull_request_review_comment` webhook event.
|
|
6
|
+
const actions = ['created', 'deleted', 'edited'] as const;
|
|
7
|
+
|
|
8
|
+
const payloadSchema = z.looseObject({
|
|
9
|
+
action: z.string(),
|
|
10
|
+
comment: z.looseObject({ id: z.number().int() }),
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export const reviewCommentFamily = defineFactFamily({
|
|
14
|
+
stream: 'facts.github.review_comment',
|
|
15
|
+
api: 'github/2022-11-28',
|
|
16
|
+
entityKind: 'github:review_comment',
|
|
17
|
+
types: buildFamilyTypes('review_comment', actions, payloadSchema),
|
|
18
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { defineFactFamily } from '@pepys/kit';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { buildFamilyTypes } from './build-family-types.ts';
|
|
4
|
+
|
|
5
|
+
// The `action` values of GitHub's `pull_request_review_thread` webhook event.
|
|
6
|
+
const actions = ['resolved', 'unresolved'] as const;
|
|
7
|
+
const commentSchema = z.looseObject({ id: z.number().int() });
|
|
8
|
+
|
|
9
|
+
const payloadSchema = z.looseObject({
|
|
10
|
+
action: z.string(),
|
|
11
|
+
thread: z.looseObject({ comments: z.array(commentSchema) }),
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
export const reviewThreadFamily = defineFactFamily({
|
|
15
|
+
stream: 'facts.github.review_thread',
|
|
16
|
+
api: 'github/2022-11-28',
|
|
17
|
+
entityKind: 'github:review_thread',
|
|
18
|
+
types: buildFamilyTypes('review_thread', actions, payloadSchema),
|
|
19
|
+
});
|
|
@@ -176,6 +176,12 @@ export interface GithubPullRequestRow {
|
|
|
176
176
|
payload: unknown;
|
|
177
177
|
}
|
|
178
178
|
|
|
179
|
+
export interface GithubPullRequestAliasRow {
|
|
180
|
+
alias: string;
|
|
181
|
+
entity_id: EntityID;
|
|
182
|
+
updated_at: string;
|
|
183
|
+
}
|
|
184
|
+
|
|
179
185
|
export interface GithubReleaseBumpRow {
|
|
180
186
|
pull_request_id: EntityID;
|
|
181
187
|
release_repository: string;
|
|
@@ -186,6 +192,10 @@ export interface GithubPullRequestKey {
|
|
|
186
192
|
id: GithubPullRequestID;
|
|
187
193
|
}
|
|
188
194
|
|
|
195
|
+
export interface GithubPullRequestAliasKey {
|
|
196
|
+
alias: string;
|
|
197
|
+
}
|
|
198
|
+
|
|
189
199
|
export interface GithubReleaseBumpKey {
|
|
190
200
|
pull_request_id: EntityID;
|
|
191
201
|
release_repository: string;
|
|
@@ -195,6 +205,8 @@ export interface GithubReleaseBumpKey {
|
|
|
195
205
|
export type GithubPullRequestOperation =
|
|
196
206
|
| { upsert: { github_pull_request: GithubPullRequestRow } }
|
|
197
207
|
| { delete: { github_pull_request: GithubPullRequestKey } }
|
|
208
|
+
| { upsert: { github_pull_request_alias: GithubPullRequestAliasRow } }
|
|
209
|
+
| { delete: { github_pull_request_alias: GithubPullRequestAliasKey } }
|
|
198
210
|
| { upsert: { github_release_bump: GithubReleaseBumpRow } }
|
|
199
211
|
| { delete: { github_release_bump: GithubReleaseBumpKey } };
|
|
200
212
|
|
|
@@ -297,6 +309,45 @@ export function readGithubPullRequestRow(key: string): GithubPullRequestRow | un
|
|
|
297
309
|
};
|
|
298
310
|
}
|
|
299
311
|
|
|
312
|
+
declare function readGithubPullRequestAliasLength(key: string): number;
|
|
313
|
+
declare function readGithubPullRequestAliasInto(key: string, row: Uint8Array): void;
|
|
314
|
+
|
|
315
|
+
export function readGithubPullRequestAliasRow(key: string): GithubPullRequestAliasRow | undefined {
|
|
316
|
+
const length = readGithubPullRequestAliasLength(key);
|
|
317
|
+
|
|
318
|
+
if (length < 0) {
|
|
319
|
+
return undefined;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const row = new Uint8Array(length);
|
|
323
|
+
|
|
324
|
+
readGithubPullRequestAliasInto(key, row);
|
|
325
|
+
|
|
326
|
+
const parsed: unknown = JSON.parse(new TextDecoder().decode(row));
|
|
327
|
+
|
|
328
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
329
|
+
return undefined;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if (!('alias' in parsed) || (typeof parsed.alias !== 'string')) {
|
|
333
|
+
return undefined;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (!('entity_id' in parsed) || (!isEntityID(parsed.entity_id))) {
|
|
337
|
+
return undefined;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (!('updated_at' in parsed) || (typeof parsed.updated_at !== 'string')) {
|
|
341
|
+
return undefined;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
return {
|
|
345
|
+
alias: parsed.alias,
|
|
346
|
+
entity_id: parsed.entity_id,
|
|
347
|
+
updated_at: parsed.updated_at,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
300
351
|
function isEntityID(value: unknown): value is EntityID {
|
|
301
352
|
return typeof value === 'string';
|
|
302
353
|
}
|
|
@@ -21,6 +21,15 @@ CREATE TABLE "github_pull_request" (
|
|
|
21
21
|
|
|
22
22
|
CREATE INDEX "github_pull_request_repository_id_idx" ON "github_pull_request" ("repository_id");
|
|
23
23
|
|
|
24
|
+
CREATE TABLE "github_pull_request_alias" (
|
|
25
|
+
"alias" TEXT NOT NULL,
|
|
26
|
+
"entity_id" TEXT NOT NULL,
|
|
27
|
+
"updated_at" TEXT NOT NULL,
|
|
28
|
+
PRIMARY KEY ("alias")
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
CREATE INDEX "github_pull_request_alias_entity_id_idx" ON "github_pull_request_alias" ("entity_id");
|
|
32
|
+
|
|
24
33
|
CREATE TABLE "github_release_bump" (
|
|
25
34
|
"pull_request_id" TEXT NOT NULL,
|
|
26
35
|
"release_repository" TEXT NOT NULL,
|
|
@@ -28,6 +28,19 @@ export const githubPullRequestConsumer = defineConsumer({
|
|
|
28
28
|
indexes: [{ columns: ['repository_id'] }],
|
|
29
29
|
}),
|
|
30
30
|
|
|
31
|
+
// Each form a person writes for a pull request: its URL and `<owner>/<repo>#<number>`, in
|
|
32
|
+
// lowercase, with the update time of the newest fact that names it. A rename adds the new forms
|
|
33
|
+
// and keeps the old ones. A form moves only to a fact newer than that time.
|
|
34
|
+
github_pull_request_alias: defineTable({
|
|
35
|
+
columns: {
|
|
36
|
+
alias: { kind: 'text' },
|
|
37
|
+
entity_id: { kind: 'entityID' },
|
|
38
|
+
updated_at: { kind: 'timestamp' },
|
|
39
|
+
},
|
|
40
|
+
primaryKey: ['alias'],
|
|
41
|
+
indexes: [{ columns: ['entity_id'] }],
|
|
42
|
+
}),
|
|
43
|
+
|
|
31
44
|
// One row per release that a bot's pull request names in its body, by the release URL. The rows
|
|
32
45
|
// follow the pull request row: a fact that loses writes none, and one that wins replaces them.
|
|
33
46
|
github_release_bump: defineTable({
|
|
@@ -51,6 +64,6 @@ export const githubPullRequestConsumer = defineConsumer({
|
|
|
51
64
|
updatedAt: 'updated_at',
|
|
52
65
|
},
|
|
53
66
|
],
|
|
54
|
-
reads: ['github_pull_request'],
|
|
67
|
+
reads: ['github_pull_request', 'github_pull_request_alias'],
|
|
55
68
|
links: [],
|
|
56
69
|
});
|
package/src/github-source.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { defineSource } from '@pepys/kit';
|
|
2
2
|
import { deploymentFamily } from './families/deployment-family.ts';
|
|
3
|
+
import { issueCommentFamily } from './families/issue-comment-family.ts';
|
|
3
4
|
import { issueFamily } from './families/issue-family.ts';
|
|
4
5
|
import { pullRequestFamily } from './families/pull-request-family.ts';
|
|
5
6
|
import { pullRequestReviewFamily } from './families/pull-request-review-family.ts';
|
|
6
7
|
import { refFamily } from './families/ref-family.ts';
|
|
7
8
|
import { releaseFamily } from './families/release-family.ts';
|
|
9
|
+
import { repositoryFamily } from './families/repository-family.ts';
|
|
10
|
+
import { reviewCommentFamily } from './families/review-comment-family.ts';
|
|
11
|
+
import { reviewThreadFamily } from './families/review-thread-family.ts';
|
|
8
12
|
import { workflowRunFamily } from './families/workflow-run-family.ts';
|
|
9
13
|
import { readGithubDelivery } from './ingest/read-github-delivery.ts';
|
|
10
14
|
import { pullFailedDeliveries } from './ingest/recovery/pull-failed-deliveries.ts';
|
|
@@ -13,7 +17,7 @@ import { pullMarkedObjects } from './ingest/resync/pull-marked-objects.ts';
|
|
|
13
17
|
// The GitHub App that both pulls call the API as.
|
|
14
18
|
const appEnv = ['PEPYS_GITHUB_APP_ID', 'PEPYS_GITHUB_APP_PRIVATE_KEY'];
|
|
15
19
|
|
|
16
|
-
// One webhook route, checked by GitHub's signature, for
|
|
20
|
+
// One webhook route, checked by GitHub's signature, for eleven families. The recovery pull reads
|
|
17
21
|
// each failed delivery every hour; the resync pull reads each marked object every minute.
|
|
18
22
|
export const githubSource = defineSource({
|
|
19
23
|
name: 'github',
|
|
@@ -25,6 +29,10 @@ export const githubSource = defineSource({
|
|
|
25
29
|
workflowRunFamily,
|
|
26
30
|
issueFamily,
|
|
27
31
|
pullRequestReviewFamily,
|
|
32
|
+
reviewCommentFamily,
|
|
33
|
+
reviewThreadFamily,
|
|
34
|
+
issueCommentFamily,
|
|
35
|
+
repositoryFamily,
|
|
28
36
|
],
|
|
29
37
|
live: {
|
|
30
38
|
kind: 'receive',
|
|
@@ -2,7 +2,7 @@ import { parseFactEnvelope } from '@pepys/kit';
|
|
|
2
2
|
import type { PendingFact, StreamName } from '@pepys/kit';
|
|
3
3
|
import { toTagged } from '@zgeoff/std';
|
|
4
4
|
import { match } from 'ts-pattern';
|
|
5
|
-
import { buildBodyHash } from '
|
|
5
|
+
import { buildBodyHash } from './build-body-hash.ts';
|
|
6
6
|
import { githubEventRules } from './github-event-rules.ts';
|
|
7
7
|
|
|
8
8
|
// `delivery`: a webhook body, from the receive path or the recovery pull. `observation`: an object
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { requestGithubAPI } from './request-github-api.ts';
|
|
3
|
+
import type { GithubCall } from './request-github-api.ts';
|
|
3
4
|
|
|
4
5
|
// `not-installed`: the App has no installation on the repository, so it can read nothing there.
|
|
5
6
|
export type InstallationToken =
|
|
@@ -14,8 +15,9 @@ const accessTokenSchema = z.looseObject({ token: z.string().min(1) });
|
|
|
14
15
|
export async function readInstallationToken(
|
|
15
16
|
repository: string,
|
|
16
17
|
appJWT: string,
|
|
18
|
+
call: GithubCall,
|
|
17
19
|
): Promise<InstallationToken> {
|
|
18
|
-
const installation = await requestGithubAPI(`/repos/${repository}/installation`, appJWT);
|
|
20
|
+
const installation = await requestGithubAPI(`/repos/${repository}/installation`, appJWT, call);
|
|
19
21
|
|
|
20
22
|
if (installation.status === 404) {
|
|
21
23
|
return { kind: 'not-installed' };
|
|
@@ -30,6 +32,7 @@ export async function readInstallationToken(
|
|
|
30
32
|
const created = await requestGithubAPI(
|
|
31
33
|
`/app/installations/${parsedInstallation.data.id}/access_tokens`,
|
|
32
34
|
appJWT,
|
|
35
|
+
call,
|
|
33
36
|
'POST',
|
|
34
37
|
);
|
|
35
38
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Logger } from '@pepys/kit';
|
|
1
2
|
import { githubAPIURL } from './github-api-url.ts';
|
|
2
3
|
|
|
3
4
|
// One answer of the GitHub REST API. `text` is the body as sent, for a caller that must read a
|
|
@@ -9,39 +10,54 @@ export interface GithubAnswer {
|
|
|
9
10
|
next: string | undefined;
|
|
10
11
|
}
|
|
11
12
|
|
|
13
|
+
// What each call needs from the pull that makes it: the logger, and the signal that aborts the call
|
|
14
|
+
// when ingest stops.
|
|
15
|
+
export interface GithubCall {
|
|
16
|
+
log: Logger;
|
|
17
|
+
signal: AbortSignal;
|
|
18
|
+
}
|
|
19
|
+
|
|
12
20
|
// One call to the GitHub REST API, with a path or a whole URL from a `link` header. A transport
|
|
13
|
-
// failure answers status 0.
|
|
21
|
+
// failure, an aborted call included, answers status 0.
|
|
14
22
|
export async function requestGithubAPI(
|
|
15
23
|
pathOrURL: string,
|
|
16
24
|
token: string,
|
|
25
|
+
call: GithubCall,
|
|
17
26
|
method: 'GET' | 'POST' = 'GET',
|
|
18
27
|
): Promise<GithubAnswer> {
|
|
19
28
|
const url = pathOrURL.startsWith('/') ? `${githubAPIURL}${pathOrURL}` : pathOrURL;
|
|
20
29
|
|
|
21
|
-
const response = await sendRequest(url, token, method);
|
|
30
|
+
const response = await sendRequest(url, token, method, call);
|
|
22
31
|
|
|
23
32
|
if (response === undefined) {
|
|
24
33
|
return { status: 0, text: '', body: undefined, next: undefined };
|
|
25
34
|
}
|
|
26
35
|
|
|
27
|
-
const text = await response.text();
|
|
28
|
-
|
|
29
36
|
return {
|
|
30
37
|
status: response.status,
|
|
31
|
-
text,
|
|
32
|
-
body: parseBody(text),
|
|
38
|
+
text: response.text,
|
|
39
|
+
body: parseBody(response.text),
|
|
33
40
|
next: findNextPage(response.headers.get('link')),
|
|
34
41
|
};
|
|
35
42
|
}
|
|
36
43
|
|
|
44
|
+
interface SentRequest {
|
|
45
|
+
status: number;
|
|
46
|
+
text: string;
|
|
47
|
+
headers: Headers;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// The body is read inside the `try`, so an abort while GitHub still sends it counts as a failed call.
|
|
37
51
|
async function sendRequest(
|
|
38
52
|
url: string,
|
|
39
53
|
token: string,
|
|
40
54
|
method: 'GET' | 'POST',
|
|
41
|
-
|
|
55
|
+
call: GithubCall,
|
|
56
|
+
): Promise<SentRequest | undefined> {
|
|
42
57
|
try {
|
|
43
58
|
const response = await fetch(url, {
|
|
44
59
|
method,
|
|
60
|
+
signal: call.signal,
|
|
45
61
|
headers: {
|
|
46
62
|
accept: 'application/vnd.github+json',
|
|
47
63
|
authorization: `Bearer ${token}`,
|
|
@@ -50,9 +66,14 @@ async function sendRequest(
|
|
|
50
66
|
},
|
|
51
67
|
});
|
|
52
68
|
|
|
53
|
-
|
|
69
|
+
const text = await response.text();
|
|
70
|
+
|
|
71
|
+
return { status: response.status, text, headers: response.headers };
|
|
54
72
|
} catch (error: unknown) {
|
|
55
|
-
|
|
73
|
+
// a call that ingest's stop aborted is no failure
|
|
74
|
+
if (!call.signal.aborted) {
|
|
75
|
+
call.log.error(`a GitHub API call to ${url} failed`, error);
|
|
76
|
+
}
|
|
56
77
|
|
|
57
78
|
return undefined;
|
|
58
79
|
}
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import type { FactFamily } from '@pepys/kit';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import { deploymentFamily } from '../families/deployment-family.ts';
|
|
4
|
+
import { issueCommentFamily } from '../families/issue-comment-family.ts';
|
|
4
5
|
import { issueFamily } from '../families/issue-family.ts';
|
|
5
6
|
import { pullRequestFamily } from '../families/pull-request-family.ts';
|
|
6
7
|
import { pullRequestReviewFamily } from '../families/pull-request-review-family.ts';
|
|
7
8
|
import { refFamily } from '../families/ref-family.ts';
|
|
8
9
|
import { releaseFamily } from '../families/release-family.ts';
|
|
10
|
+
import { repositoryFamily } from '../families/repository-family.ts';
|
|
11
|
+
import { reviewCommentFamily } from '../families/review-comment-family.ts';
|
|
12
|
+
import { reviewThreadFamily } from '../families/review-thread-family.ts';
|
|
9
13
|
import { workflowRunFamily } from '../families/workflow-run-family.ts';
|
|
10
14
|
|
|
11
15
|
// What a rule reads from one delivery body. `locator` is the API path the resync pull reads the
|
|
@@ -95,6 +99,37 @@ const reviewBodySchema = z.looseObject({
|
|
|
95
99
|
sender: senderSchema,
|
|
96
100
|
});
|
|
97
101
|
|
|
102
|
+
const commentBodySchema = z.looseObject({
|
|
103
|
+
action: z.string().min(1),
|
|
104
|
+
comment: z.looseObject({ id: z.number().int(), updated_at: timeSchema }),
|
|
105
|
+
sender: senderSchema,
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
const threadCommentSchema = z.looseObject({
|
|
109
|
+
id: z.number().int(),
|
|
110
|
+
in_reply_to_id: z.number().int().nullish(),
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const reviewThreadBodySchema = z.looseObject({
|
|
114
|
+
action: z.string().min(1),
|
|
115
|
+
thread: z.looseObject({ comments: z.array(threadCommentSchema) }),
|
|
116
|
+
updated_at: timeSchema.nullish(),
|
|
117
|
+
pull_request: z.looseObject({ updated_at: timeSchema }),
|
|
118
|
+
sender: senderSchema,
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
const secondsOrTimeSchema = z.union([z.number().int(), timeSchema]);
|
|
122
|
+
|
|
123
|
+
const repositoryBodySchema = z.looseObject({
|
|
124
|
+
action: z.string().min(1),
|
|
125
|
+
repository: z.looseObject({
|
|
126
|
+
id: z.number().int(),
|
|
127
|
+
created_at: secondsOrTimeSchema.nullish(),
|
|
128
|
+
updated_at: timeSchema.nullish(),
|
|
129
|
+
}),
|
|
130
|
+
sender: senderSchema,
|
|
131
|
+
});
|
|
132
|
+
|
|
98
133
|
// One rule per webhook event name. Any other event is ignored.
|
|
99
134
|
export const githubEventRules: Readonly<Record<string, GithubEventRule>> = {
|
|
100
135
|
pull_request: {
|
|
@@ -261,6 +296,82 @@ export const githubEventRules: Readonly<Record<string, GithubEventRule>> = {
|
|
|
261
296
|
);
|
|
262
297
|
},
|
|
263
298
|
},
|
|
299
|
+
pull_request_review_comment: {
|
|
300
|
+
family: reviewCommentFamily,
|
|
301
|
+
read: (body) => {
|
|
302
|
+
const parsed = commentBodySchema.safeParse(body);
|
|
303
|
+
|
|
304
|
+
if (!parsed.success) {
|
|
305
|
+
return 'invalid';
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
return pickFields(reviewCommentFamily, `github.review_comment.${parsed.data.action}`, {
|
|
309
|
+
entity: `github:review_comment:${parsed.data.comment.id}`,
|
|
310
|
+
actor: parsed.data.sender,
|
|
311
|
+
time: parsed.data.comment.updated_at,
|
|
312
|
+
locator: undefined,
|
|
313
|
+
});
|
|
314
|
+
},
|
|
315
|
+
},
|
|
316
|
+
pull_request_review_thread: {
|
|
317
|
+
family: reviewThreadFamily,
|
|
318
|
+
read: (body) => {
|
|
319
|
+
const parsed = reviewThreadBodySchema.safeParse(body);
|
|
320
|
+
|
|
321
|
+
if (!parsed.success) {
|
|
322
|
+
return 'invalid';
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
const firstCommentID = findFirstCommentID(parsed.data.thread.comments);
|
|
326
|
+
|
|
327
|
+
if (firstCommentID === undefined) {
|
|
328
|
+
return 'invalid';
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
return pickFields(reviewThreadFamily, `github.review_thread.${parsed.data.action}`, {
|
|
332
|
+
entity: `github:review_thread:${firstCommentID}`,
|
|
333
|
+
actor: parsed.data.sender,
|
|
334
|
+
time: parsed.data.updated_at ?? parsed.data.pull_request.updated_at,
|
|
335
|
+
locator: undefined,
|
|
336
|
+
});
|
|
337
|
+
},
|
|
338
|
+
},
|
|
339
|
+
issue_comment: {
|
|
340
|
+
family: issueCommentFamily,
|
|
341
|
+
read: (body) => {
|
|
342
|
+
const parsed = commentBodySchema.safeParse(body);
|
|
343
|
+
|
|
344
|
+
if (!parsed.success) {
|
|
345
|
+
return 'invalid';
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return pickFields(issueCommentFamily, `github.issue_comment.${parsed.data.action}`, {
|
|
349
|
+
entity: `github:issue_comment:${parsed.data.comment.id}`,
|
|
350
|
+
actor: parsed.data.sender,
|
|
351
|
+
time: parsed.data.comment.updated_at,
|
|
352
|
+
locator: undefined,
|
|
353
|
+
});
|
|
354
|
+
},
|
|
355
|
+
},
|
|
356
|
+
repository: {
|
|
357
|
+
family: repositoryFamily,
|
|
358
|
+
read: (body) => {
|
|
359
|
+
const parsed = repositoryBodySchema.safeParse(body);
|
|
360
|
+
|
|
361
|
+
if (!parsed.success) {
|
|
362
|
+
return 'invalid';
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const repository = parsed.data.repository;
|
|
366
|
+
|
|
367
|
+
return pickFields(repositoryFamily, `github.repository.${parsed.data.action}`, {
|
|
368
|
+
entity: `github:repo:${repository.id}`,
|
|
369
|
+
actor: parsed.data.sender,
|
|
370
|
+
time: repository.updated_at ?? toRepositoryTime(repository.created_at),
|
|
371
|
+
locator: undefined,
|
|
372
|
+
});
|
|
373
|
+
},
|
|
374
|
+
},
|
|
264
375
|
};
|
|
265
376
|
|
|
266
377
|
interface RuleValues {
|
|
@@ -322,3 +433,20 @@ function toAPIPath(url: string | undefined): string | undefined {
|
|
|
322
433
|
|
|
323
434
|
return path.startsWith('/repos/') ? path : undefined;
|
|
324
435
|
}
|
|
436
|
+
|
|
437
|
+
// GitHub points every reply at the first comment of its thread, so a thread whose first comment
|
|
438
|
+
// was deleted still gives that comment's ID.
|
|
439
|
+
function findFirstCommentID(
|
|
440
|
+
comments: readonly z.infer<typeof threadCommentSchema>[],
|
|
441
|
+
): number | undefined {
|
|
442
|
+
const first = comments.find(
|
|
443
|
+
(comment) => comment.in_reply_to_id === undefined || comment.in_reply_to_id === null,
|
|
444
|
+
);
|
|
445
|
+
|
|
446
|
+
return first?.id ?? comments[0]?.in_reply_to_id ?? undefined;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// GitHub's repository schema allows a repository's `created_at` as Unix seconds as well as text.
|
|
450
|
+
function toRepositoryTime(time: number | string | null | undefined): string | null | undefined {
|
|
451
|
+
return typeof time === 'number' ? new Date(time * 1000).toISOString() : time;
|
|
452
|
+
}
|
|
@@ -5,7 +5,7 @@ import { buildGithubFact } from '../build-github-fact.ts';
|
|
|
5
5
|
import { findAppJWT } from '../github-app/find-app-jwt.ts';
|
|
6
6
|
import { pickPullFailure } from '../github-app/pick-pull-failure.ts';
|
|
7
7
|
import { requestGithubAPI } from '../github-app/request-github-api.ts';
|
|
8
|
-
import type { GithubAnswer } from '../github-app/request-github-api.ts';
|
|
8
|
+
import type { GithubAnswer, GithubCall } from '../github-app/request-github-api.ts';
|
|
9
9
|
|
|
10
10
|
const deliveryDetailSchema = z.looseObject({
|
|
11
11
|
event: z.string(),
|
|
@@ -27,7 +27,7 @@ export async function pullFailedDeliveries(
|
|
|
27
27
|
|
|
28
28
|
const seenID = cursor === null ? 0n : BigInt(cursor);
|
|
29
29
|
|
|
30
|
-
const listed = await readDeliveriesAfter(seenID, appJWT);
|
|
30
|
+
const listed = await readDeliveriesAfter(seenID, appJWT, context);
|
|
31
31
|
|
|
32
32
|
if (listed.kind !== 'listed') {
|
|
33
33
|
return pickPullFailure(listed.status);
|
|
@@ -36,7 +36,7 @@ export async function pullFailedDeliveries(
|
|
|
36
36
|
const facts: PendingFact[] = [];
|
|
37
37
|
|
|
38
38
|
for (const id of collectFailedIDs(listed.items)) {
|
|
39
|
-
const answer = await requestGithubAPI(`/app/hook/deliveries/${id}`, appJWT);
|
|
39
|
+
const answer = await requestGithubAPI(`/app/hook/deliveries/${id}`, appJWT, context);
|
|
40
40
|
|
|
41
41
|
const detail = deliveryDetailSchema.safeParse(answer.body);
|
|
42
42
|
|
|
@@ -55,7 +55,7 @@ export async function pullFailedDeliveries(
|
|
|
55
55
|
// The receive path answers 413 to a body this large and stores nothing; a batch that held it
|
|
56
56
|
// would be refused whole, with every other delivery and the cursor.
|
|
57
57
|
if (!planAppends([fact.fact]).ok) {
|
|
58
|
-
|
|
58
|
+
context.log.error(`the GitHub delivery ${id} is larger than one append of the log takes`);
|
|
59
59
|
continue;
|
|
60
60
|
}
|
|
61
61
|
|
|
@@ -87,12 +87,16 @@ type DeliveryItem = z.infer<typeof deliveryItemSchema>;
|
|
|
87
87
|
type DeliveryList = { kind: 'listed'; items: DeliveryItem[] } | { kind: 'failed'; status: number };
|
|
88
88
|
|
|
89
89
|
// GitHub lists the newest delivery first, so the pages stop at the first one already seen.
|
|
90
|
-
async function readDeliveriesAfter(
|
|
90
|
+
async function readDeliveriesAfter(
|
|
91
|
+
seenID: bigint,
|
|
92
|
+
appJWT: string,
|
|
93
|
+
call: GithubCall,
|
|
94
|
+
): Promise<DeliveryList> {
|
|
91
95
|
const items: DeliveryItem[] = [];
|
|
92
96
|
let url: string | undefined = '/app/hook/deliveries?per_page=100';
|
|
93
97
|
|
|
94
98
|
while (url !== undefined) {
|
|
95
|
-
const answer: GithubAnswer = await requestGithubAPI(url, appJWT);
|
|
99
|
+
const answer: GithubAnswer = await requestGithubAPI(url, appJWT, call);
|
|
96
100
|
|
|
97
101
|
const page = deliveryListSchema.safeParse(parseDeliveryList(answer.text));
|
|
98
102
|
|
|
@@ -7,6 +7,7 @@ import { pickPullFailure } from '../github-app/pick-pull-failure.ts';
|
|
|
7
7
|
import { readInstallationToken } from '../github-app/read-installation-token.ts';
|
|
8
8
|
import type { InstallationToken } from '../github-app/read-installation-token.ts';
|
|
9
9
|
import { requestGithubAPI } from '../github-app/request-github-api.ts';
|
|
10
|
+
import type { GithubCall } from '../github-app/request-github-api.ts';
|
|
10
11
|
import { collectResyncTargets } from './collect-resync-targets.ts';
|
|
11
12
|
import type { ResyncTarget } from './collect-resync-targets.ts';
|
|
12
13
|
|
|
@@ -34,7 +35,7 @@ export async function pullMarkedObjects(
|
|
|
34
35
|
const facts: PendingFact[] = [];
|
|
35
36
|
|
|
36
37
|
for (const target of targets) {
|
|
37
|
-
const token = await readCachedToken(tokens, target.repository, appJWT);
|
|
38
|
+
const token = await readCachedToken(tokens, target.repository, appJWT, context);
|
|
38
39
|
|
|
39
40
|
if (token.kind === 'failed') {
|
|
40
41
|
return pickPullFailure(token.status);
|
|
@@ -44,7 +45,7 @@ export async function pullMarkedObjects(
|
|
|
44
45
|
continue;
|
|
45
46
|
}
|
|
46
47
|
|
|
47
|
-
const read = await readObject(target, token.token);
|
|
48
|
+
const read = await readObject(target, token.token, context);
|
|
48
49
|
|
|
49
50
|
if (read.kind === 'failed') {
|
|
50
51
|
return pickPullFailure(read.status);
|
|
@@ -63,6 +64,7 @@ async function readCachedToken(
|
|
|
63
64
|
tokens: Map<string, InstallationToken>,
|
|
64
65
|
repository: string,
|
|
65
66
|
appJWT: string,
|
|
67
|
+
call: GithubCall,
|
|
66
68
|
): Promise<InstallationToken> {
|
|
67
69
|
const known = tokens.get(repository);
|
|
68
70
|
|
|
@@ -70,7 +72,7 @@ async function readCachedToken(
|
|
|
70
72
|
return known;
|
|
71
73
|
}
|
|
72
74
|
|
|
73
|
-
const token = await readInstallationToken(repository, appJWT);
|
|
75
|
+
const token = await readInstallationToken(repository, appJWT, call);
|
|
74
76
|
|
|
75
77
|
tokens.set(repository, token);
|
|
76
78
|
|
|
@@ -87,8 +89,12 @@ type ReadObject =
|
|
|
87
89
|
// An object the API answers 404 for is left alone. GitHub deletes no pull request, and shows a draft
|
|
88
90
|
// release only to a token that can push, so a 404 cannot prove a delete. A deleted release's own
|
|
89
91
|
// `deleted` delivery sets its tombstone.
|
|
90
|
-
async function readObject(
|
|
91
|
-
|
|
92
|
+
async function readObject(
|
|
93
|
+
target: ResyncTarget,
|
|
94
|
+
token: string,
|
|
95
|
+
call: GithubCall,
|
|
96
|
+
): Promise<ReadObject> {
|
|
97
|
+
const answer = await requestGithubAPI(target.path, token, call);
|
|
92
98
|
|
|
93
99
|
if (answer.status === 404) {
|
|
94
100
|
return { kind: 'none' };
|
|
@@ -106,7 +112,7 @@ async function readObject(target: ResyncTarget, token: string): Promise<ReadObje
|
|
|
106
112
|
);
|
|
107
113
|
}
|
|
108
114
|
|
|
109
|
-
const repository = await requestGithubAPI(`/repos/${target.repository}`, token);
|
|
115
|
+
const repository = await requestGithubAPI(`/repos/${target.repository}`, token, call);
|
|
110
116
|
|
|
111
117
|
const parsed = repositorySchema.safeParse(repository.body);
|
|
112
118
|
|
|
File without changes
|