@lobu/runtime-device 21.0.0 → 21.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +5 -5
- package/npm-shrinkwrap.json +23 -23
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +0 -4
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +1 -4
- package/vendor/cli/dist/commands/_lib/apply/client.js +2 -8
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +0 -1
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +0 -2
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +0 -1
- package/vendor/cli/dist/commands/_lib/apply/diff.js +0 -6
- package/vendor/cli/dist/commands/_lib/view-bundler.js +6 -2
- package/vendor/cli/dist/commands/memory/_lib/schema.d.ts +0 -1
- package/vendor/cli/dist/commands/memory/_lib/schema.js +0 -9
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +0 -2
- package/vendor/cli/dist/connectors/github.ts +269 -194
- package/vendor/cli/dist/connectors/google_calendar.ts +36 -132
- package/vendor/cli/dist/connectors/jira.ts +13 -4
- package/vendor/cli/dist/connectors/producthunt.ts +3 -1
- package/vendor/cli/dist/connectors/reddit.ts +34 -4
- package/vendor/cli/dist/providers.json +0 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +1 -1
- package/vendor/connector-sdk/dist/connector-types.d.ts +0 -2
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/daemon/client.d.ts +4 -1
- package/vendor/connector-worker/dist/daemon/client.js +3 -0
- package/vendor/connector-worker/dist/daemon/executor.js +4 -0
- package/vendor/connector-worker/dist/daemon/start.js +2 -2
- package/vendor/connector-worker/dist/executor/interface.d.ts +7 -1
- package/vendor/connector-worker/dist/executor/isolate.js +40 -0
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-automations.d.ts +0 -10
- package/vendor/core/dist/contracts/tools/manage-automations.js +0 -33
- package/vendor/core/dist/contracts/tools/manage-classifiers.d.ts +4 -34
- package/vendor/core/dist/contracts/tools/manage-classifiers.js +13 -44
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +0 -9
- package/vendor/core/dist/contracts/tools/manage-entity.js +0 -6
- package/vendor/core/dist/contracts/tools/view-attach.d.ts +20 -0
- package/vendor/core/dist/contracts/tools/view-attach.js +45 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +17 -0
- package/vendor/core/dist/contracts/worker/protocol.js +12 -1
- package/vendor/core/dist/index.d.ts +1 -1
- package/vendor/core/dist/provider-config-types.d.ts +0 -9
- package/vendor/core/dist/sdk-compat.d.ts +1 -1
- package/vendor/core/package.json +6 -1
|
@@ -17,7 +17,6 @@ import {
|
|
|
17
17
|
createHttpClient,
|
|
18
18
|
type EventAttributionRule,
|
|
19
19
|
type EventEnvelope,
|
|
20
|
-
paginateByOffset,
|
|
21
20
|
SubscriptionCandidateSchema,
|
|
22
21
|
type SyncContext,
|
|
23
22
|
type SyncResult,
|
|
@@ -75,9 +74,50 @@ interface GitHubConfig {
|
|
|
75
74
|
installation_ref?: number | string;
|
|
76
75
|
}
|
|
77
76
|
|
|
77
|
+
interface GitHubPosition {
|
|
78
|
+
page?: number;
|
|
79
|
+
cursor?: string;
|
|
80
|
+
comment_cursor?: string;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface GitHubContentProgress extends GitHubPosition {
|
|
84
|
+
scope: string;
|
|
85
|
+
since: string;
|
|
86
|
+
started_at: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
interface GitHubPage {
|
|
90
|
+
events: EventEnvelope[];
|
|
91
|
+
next?: GitHubPosition;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
interface GitHubPageInfo {
|
|
95
|
+
hasNextPage: boolean;
|
|
96
|
+
endCursor?: string | null;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function nextGitHubCursor(info: GitHubPageInfo | undefined, current?: string): string | undefined {
|
|
100
|
+
if (!info || typeof info.hasNextPage !== 'boolean') {
|
|
101
|
+
throw new Error('GitHub omitted pagination information.');
|
|
102
|
+
}
|
|
103
|
+
if (!info.hasNextPage) return undefined;
|
|
104
|
+
if (!info.endCursor || info.endCursor === current) {
|
|
105
|
+
throw new Error('GitHub returned a missing or repeated page cursor.');
|
|
106
|
+
}
|
|
107
|
+
return info.endCursor;
|
|
108
|
+
}
|
|
109
|
+
|
|
78
110
|
interface GitHubCheckpoint {
|
|
79
111
|
last_sync_at?: string;
|
|
112
|
+
commits?: {
|
|
113
|
+
since: string;
|
|
114
|
+
started_at: string;
|
|
115
|
+
head_sha?: string;
|
|
116
|
+
next_page: number;
|
|
117
|
+
};
|
|
118
|
+
content?: GitHubContentProgress;
|
|
80
119
|
stargazers?: GitHubStargazerCheckpoint[];
|
|
120
|
+
stargazer_scan?: { scope: string; page: number; started_at: string; current: GitHubStargazerCheckpoint[] };
|
|
81
121
|
}
|
|
82
122
|
|
|
83
123
|
interface GitHubStargazerCheckpoint {
|
|
@@ -243,18 +283,6 @@ function asString(value: unknown): string | undefined {
|
|
|
243
283
|
return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
|
|
244
284
|
}
|
|
245
285
|
|
|
246
|
-
/**
|
|
247
|
-
* Coerce a GitHub list response to an array. `http.json()` returns
|
|
248
|
-
* `await response.json()`, which is `null`/`undefined` for an empty-body 200
|
|
249
|
-
* (GitHub occasionally serves these). The list sync paths immediately read
|
|
250
|
-
* `.length` / iterate, so a bare null threw
|
|
251
|
-
* `Cannot read properties of undefined (reading 'length')` and crashed the
|
|
252
|
-
* whole sync (#2033). Treat any non-array response as an empty page.
|
|
253
|
-
*/
|
|
254
|
-
function asArray<T>(value: T[] | null | undefined): T[] {
|
|
255
|
-
return Array.isArray(value) ? value : [];
|
|
256
|
-
}
|
|
257
|
-
|
|
258
286
|
function toIsoOrUndefined(value: unknown): string | undefined {
|
|
259
287
|
const str = asString(value);
|
|
260
288
|
if (!str) return undefined;
|
|
@@ -280,9 +308,7 @@ const LOOKBACK_PROP = {
|
|
|
280
308
|
const STARGAZER_PROFILE_REFRESH_MS = 30 * 24 * 60 * 60 * 1000;
|
|
281
309
|
const STARGAZER_PROFILE_FETCH_LIMIT = 25;
|
|
282
310
|
|
|
283
|
-
//
|
|
284
|
-
// keeps steady-state runs tiny; the cap only matters on a cold backfill of a
|
|
285
|
-
// busy repo, where the remaining history rides subsequent runs.
|
|
311
|
+
// Resume after at most 30 data pages, keeping the same immutable commit graph.
|
|
286
312
|
const COMMITS_MAX_PAGES = 30;
|
|
287
313
|
|
|
288
314
|
const LABELS_PROP = {
|
|
@@ -902,37 +928,37 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
902
928
|
);
|
|
903
929
|
const sinceIso = this.resolveSince(ctx.checkpoint, config.lookback_days ?? 365);
|
|
904
930
|
|
|
931
|
+
const checkpoint = (ctx.checkpoint ?? {}) as GitHubCheckpoint;
|
|
932
|
+
let result: { events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] };
|
|
905
933
|
if (contentType === 'stargazers') {
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
934
|
+
result = await this.syncStargazers(repo, checkpoint, token);
|
|
935
|
+
} else if (contentType === 'commits') {
|
|
936
|
+
result = await this.syncCommits(repo, sinceIso, ctx.checkpoint, token);
|
|
937
|
+
} else {
|
|
938
|
+
const labelsFilter = config.labels_filter ?? [];
|
|
939
|
+
const scope = JSON.stringify([repo.owner, repo.repo, contentType, labelsFilter, config.lookback_days ?? 365]);
|
|
940
|
+
const progress = checkpoint.content?.scope === scope ? checkpoint.content : {
|
|
941
|
+
scope,
|
|
942
|
+
since: sinceIso,
|
|
943
|
+
// The REST `since` boundary is exclusive and timestamps have second
|
|
944
|
+
// precision. Overlap that second to recover updates made during a scan.
|
|
945
|
+
started_at: new Date(Math.floor(Date.now() / 1000) * 1000 - 1000).toISOString(),
|
|
946
|
+
};
|
|
947
|
+
const page = await this.syncContent({ repo, contentType, progress, labelsFilter, token });
|
|
948
|
+
result = {
|
|
949
|
+
events: page.events,
|
|
950
|
+
checkpoint: page.next
|
|
951
|
+
? { ...checkpoint, content: { scope, since: progress.since, started_at: progress.started_at, ...page.next } }
|
|
952
|
+
: { last_sync_at: progress.started_at },
|
|
953
|
+
status: page.next ? 'more' : 'complete',
|
|
919
954
|
};
|
|
920
955
|
}
|
|
921
|
-
|
|
922
|
-
const events = await this.syncContent({
|
|
923
|
-
repo,
|
|
924
|
-
contentType,
|
|
925
|
-
sinceIso,
|
|
926
|
-
labelsFilter: config.labels_filter ?? [],
|
|
927
|
-
token,
|
|
928
|
-
});
|
|
956
|
+
const { events } = result;
|
|
929
957
|
this.stampRepoAttribution(events, repo, { attachAutomationSignals });
|
|
930
958
|
|
|
931
|
-
await ctx.commit(events,
|
|
932
|
-
last_sync_at: new Date().toISOString(),
|
|
933
|
-
} as Record<string, unknown>);
|
|
959
|
+
await ctx.commit(events, result.checkpoint as Record<string, unknown>);
|
|
934
960
|
return {
|
|
935
|
-
status:
|
|
961
|
+
status: result.status,
|
|
936
962
|
metadata: {
|
|
937
963
|
items_found: events.length,
|
|
938
964
|
},
|
|
@@ -1157,58 +1183,83 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1157
1183
|
|
|
1158
1184
|
private async syncContent(params: {
|
|
1159
1185
|
repo: RepoRef;
|
|
1160
|
-
contentType: Exclude<GitHubContentType, 'stargazers'>;
|
|
1161
|
-
|
|
1186
|
+
contentType: Exclude<GitHubContentType, 'stargazers' | 'commits'>;
|
|
1187
|
+
progress: GitHubContentProgress;
|
|
1162
1188
|
labelsFilter: string[];
|
|
1163
1189
|
token: string | null;
|
|
1164
|
-
}): Promise<
|
|
1165
|
-
const { repo, contentType,
|
|
1190
|
+
}): Promise<GitHubPage> {
|
|
1191
|
+
const { repo, contentType, progress, labelsFilter, token } = params;
|
|
1166
1192
|
|
|
1167
1193
|
switch (contentType) {
|
|
1168
1194
|
case 'issues':
|
|
1169
1195
|
case 'pull_requests':
|
|
1170
|
-
return await this.syncIssuesAndPulls(repo, contentType,
|
|
1196
|
+
return await this.syncIssuesAndPulls(repo, contentType, progress, labelsFilter, token);
|
|
1171
1197
|
case 'issue_comments':
|
|
1172
|
-
return await this.syncIssueComments(repo,
|
|
1198
|
+
return await this.syncIssueComments(repo, progress, token);
|
|
1173
1199
|
case 'pr_comments':
|
|
1174
|
-
return await this.syncPullRequestComments(repo,
|
|
1200
|
+
return await this.syncPullRequestComments(repo, progress, token);
|
|
1175
1201
|
case 'discussions':
|
|
1176
|
-
return await this.syncDiscussions(repo,
|
|
1202
|
+
return await this.syncDiscussions(repo, progress, token);
|
|
1177
1203
|
case 'discussion_comments':
|
|
1178
|
-
return await this.syncDiscussionComments(repo,
|
|
1179
|
-
case 'commits':
|
|
1180
|
-
return await this.syncCommits(repo, sinceIso, token);
|
|
1204
|
+
return await this.syncDiscussionComments(repo, progress, token);
|
|
1181
1205
|
}
|
|
1182
1206
|
}
|
|
1183
1207
|
|
|
1184
1208
|
private async syncCommits(
|
|
1185
1209
|
repo: RepoRef,
|
|
1186
1210
|
sinceIso: string,
|
|
1211
|
+
rawCheckpoint: Record<string, unknown> | null,
|
|
1187
1212
|
token: string | null
|
|
1188
|
-
): Promise<EventEnvelope[]> {
|
|
1213
|
+
): Promise<{ events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] }> {
|
|
1189
1214
|
const events: EventEnvelope[] = [];
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
// walk an unbounded history (deeper backfill rides subsequent runs).
|
|
1195
|
-
const pages = paginateByOffset<GitHubCommitLike>(
|
|
1196
|
-
async (offset, pageSize) => {
|
|
1197
|
-
const query = new URLSearchParams({
|
|
1198
|
-
per_page: String(pageSize),
|
|
1199
|
-
page: String(offset / pageSize + 1),
|
|
1215
|
+
const checkpoint = (rawCheckpoint ?? {}) as GitHubCheckpoint;
|
|
1216
|
+
const progress: NonNullable<GitHubCheckpoint['commits']> = checkpoint.commits
|
|
1217
|
+
? { ...checkpoint.commits }
|
|
1218
|
+
: {
|
|
1200
1219
|
since: sinceIso,
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
{
|
|
1209
|
-
|
|
1220
|
+
// Use the start, not the end of a potentially multi-run scan. Round down
|
|
1221
|
+
// to GitHub's timestamp precision so commits in this second are replayed.
|
|
1222
|
+
started_at: new Date(Math.floor(Date.now() / 1000) * 1000).toISOString(),
|
|
1223
|
+
next_page: 1,
|
|
1224
|
+
};
|
|
1225
|
+
const more = () => ({
|
|
1226
|
+
events,
|
|
1227
|
+
checkpoint: { ...checkpoint, commits: progress },
|
|
1228
|
+
status: 'more' as const,
|
|
1229
|
+
});
|
|
1230
|
+
const complete = () => ({
|
|
1231
|
+
events,
|
|
1232
|
+
checkpoint: { last_sync_at: progress.started_at },
|
|
1233
|
+
status: 'complete' as const,
|
|
1234
|
+
});
|
|
1210
1235
|
|
|
1211
|
-
|
|
1236
|
+
if (!progress.head_sha) {
|
|
1237
|
+
// Resolve the unfiltered default-branch head before paging. A date-filtered
|
|
1238
|
+
// first commit need not be the tip; pinning that could omit merge ancestry.
|
|
1239
|
+
const head = await this.requestJson<GitHubCommitLike[]>({
|
|
1240
|
+
url: `${githubRepoApiUrl(repo, '/commits')}?per_page=1`,
|
|
1241
|
+
token,
|
|
1242
|
+
});
|
|
1243
|
+
if (!Array.isArray(head)) return more();
|
|
1244
|
+
if (head.length === 0) return complete();
|
|
1245
|
+
progress.head_sha = head[0]?.sha;
|
|
1246
|
+
if (!progress.head_sha) return more();
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
for (let page = 0; page < COMMITS_MAX_PAGES; page++) {
|
|
1250
|
+
const query = new URLSearchParams({
|
|
1251
|
+
per_page: '100',
|
|
1252
|
+
page: String(progress.next_page),
|
|
1253
|
+
since: progress.since,
|
|
1254
|
+
// New commits and force pushes cannot move offsets in this snapshot.
|
|
1255
|
+
sha: progress.head_sha,
|
|
1256
|
+
});
|
|
1257
|
+
const commits = await this.requestJson<GitHubCommitLike[]>({
|
|
1258
|
+
url: `${githubRepoApiUrl(repo, '/commits')}?${query.toString()}`,
|
|
1259
|
+
token,
|
|
1260
|
+
});
|
|
1261
|
+
// An empty body is indeterminate, not evidence that history is exhausted.
|
|
1262
|
+
if (!Array.isArray(commits)) return more();
|
|
1212
1263
|
for (const commit of commits) {
|
|
1213
1264
|
if (!commit.sha) continue;
|
|
1214
1265
|
const authoredAtIso = commit.commit?.author?.date;
|
|
@@ -1242,31 +1293,36 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1242
1293
|
},
|
|
1243
1294
|
});
|
|
1244
1295
|
}
|
|
1296
|
+
progress.next_page += 1;
|
|
1297
|
+
if (commits.length < 100) return complete();
|
|
1245
1298
|
}
|
|
1246
1299
|
|
|
1247
|
-
return
|
|
1300
|
+
return more();
|
|
1248
1301
|
}
|
|
1249
1302
|
|
|
1250
1303
|
private async syncIssuesAndPulls(
|
|
1251
1304
|
repo: RepoRef,
|
|
1252
1305
|
contentType: 'issues' | 'pull_requests',
|
|
1253
|
-
|
|
1306
|
+
progress: GitHubContentProgress,
|
|
1254
1307
|
labelsFilter: string[],
|
|
1255
1308
|
token: string | null
|
|
1256
|
-
): Promise<
|
|
1309
|
+
): Promise<GitHubPage> {
|
|
1257
1310
|
const query = new URLSearchParams({
|
|
1258
1311
|
state: 'all',
|
|
1259
1312
|
per_page: '100',
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1313
|
+
// Updates must not move consumed records to a different offset.
|
|
1314
|
+
sort: 'created',
|
|
1315
|
+
direction: 'asc',
|
|
1316
|
+
since: progress.since,
|
|
1317
|
+
page: String(progress.page ?? 1),
|
|
1263
1318
|
});
|
|
1264
1319
|
if (labelsFilter.length > 0) {
|
|
1265
1320
|
query.set('labels', labelsFilter.join(','));
|
|
1266
1321
|
}
|
|
1267
1322
|
|
|
1268
1323
|
const url = `${githubRepoApiUrl(repo, '/issues')}?${query.toString()}`;
|
|
1269
|
-
const items =
|
|
1324
|
+
const items = await this.requestJson<GitHubIssueLike[]>({ url, token });
|
|
1325
|
+
if (!Array.isArray(items)) return { events: [], next: progress };
|
|
1270
1326
|
const events: EventEnvelope[] = [];
|
|
1271
1327
|
|
|
1272
1328
|
for (const item of items) {
|
|
@@ -1303,26 +1359,27 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1303
1359
|
});
|
|
1304
1360
|
}
|
|
1305
1361
|
|
|
1306
|
-
return events;
|
|
1362
|
+
return { events, next: items.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
|
|
1307
1363
|
}
|
|
1308
1364
|
|
|
1309
1365
|
private async syncIssueComments(
|
|
1310
1366
|
repo: RepoRef,
|
|
1311
|
-
|
|
1367
|
+
progress: GitHubContentProgress,
|
|
1312
1368
|
token: string | null
|
|
1313
|
-
): Promise<
|
|
1369
|
+
): Promise<GitHubPage> {
|
|
1314
1370
|
const query = new URLSearchParams({
|
|
1315
1371
|
per_page: '100',
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1372
|
+
// Updates must not move consumed records to a different offset.
|
|
1373
|
+
sort: 'created',
|
|
1374
|
+
direction: 'asc',
|
|
1375
|
+
since: progress.since,
|
|
1376
|
+
page: String(progress.page ?? 1),
|
|
1319
1377
|
});
|
|
1320
1378
|
const url = `${githubRepoApiUrl(repo, '/issues/comments')}?${query.toString()}`;
|
|
1321
|
-
const comments =
|
|
1322
|
-
|
|
1323
|
-
);
|
|
1379
|
+
const comments = await this.requestJson<GitHubCommentLike[]>({ url, token });
|
|
1380
|
+
if (!Array.isArray(comments)) return { events: [], next: progress };
|
|
1324
1381
|
|
|
1325
|
-
|
|
1382
|
+
const events = comments
|
|
1326
1383
|
.map((comment): EventEnvelope | null => {
|
|
1327
1384
|
const createdAt = new Date(comment.created_at);
|
|
1328
1385
|
if (Number.isNaN(createdAt.getTime())) return null;
|
|
@@ -1348,25 +1405,27 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1348
1405
|
};
|
|
1349
1406
|
})
|
|
1350
1407
|
.filter((value): value is EventEnvelope => value !== null);
|
|
1408
|
+
return { events, next: comments.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
|
|
1351
1409
|
}
|
|
1352
1410
|
|
|
1353
1411
|
private async syncPullRequestComments(
|
|
1354
1412
|
repo: RepoRef,
|
|
1355
|
-
|
|
1413
|
+
progress: GitHubContentProgress,
|
|
1356
1414
|
token: string | null
|
|
1357
|
-
): Promise<
|
|
1415
|
+
): Promise<GitHubPage> {
|
|
1358
1416
|
const query = new URLSearchParams({
|
|
1359
1417
|
per_page: '100',
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1418
|
+
// Updates must not move consumed records to a different offset.
|
|
1419
|
+
sort: 'created',
|
|
1420
|
+
direction: 'asc',
|
|
1421
|
+
since: progress.since,
|
|
1422
|
+
page: String(progress.page ?? 1),
|
|
1363
1423
|
});
|
|
1364
1424
|
const url = `${githubRepoApiUrl(repo, '/pulls/comments')}?${query.toString()}`;
|
|
1365
|
-
const comments =
|
|
1366
|
-
|
|
1367
|
-
);
|
|
1425
|
+
const comments = await this.requestJson<GitHubCommentLike[]>({ url, token });
|
|
1426
|
+
if (!Array.isArray(comments)) return { events: [], next: progress };
|
|
1368
1427
|
|
|
1369
|
-
|
|
1428
|
+
const events = comments
|
|
1370
1429
|
.map((comment): EventEnvelope | null => {
|
|
1371
1430
|
const createdAt = new Date(comment.created_at);
|
|
1372
1431
|
if (Number.isNaN(createdAt.getTime())) return null;
|
|
@@ -1392,17 +1451,19 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1392
1451
|
};
|
|
1393
1452
|
})
|
|
1394
1453
|
.filter((value): value is EventEnvelope => value !== null);
|
|
1454
|
+
return { events, next: comments.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
|
|
1395
1455
|
}
|
|
1396
1456
|
|
|
1397
1457
|
private async syncDiscussions(
|
|
1398
1458
|
repo: RepoRef,
|
|
1399
|
-
|
|
1459
|
+
progress: GitHubContentProgress,
|
|
1400
1460
|
token: string | null
|
|
1401
|
-
): Promise<
|
|
1461
|
+
): Promise<GitHubPage> {
|
|
1402
1462
|
const query = `
|
|
1403
|
-
query($owner: String!, $repo: String
|
|
1463
|
+
query($owner: String!, $repo: String!, $cursor: String) {
|
|
1404
1464
|
repository(owner: $owner, name: $repo) {
|
|
1405
|
-
discussions(first: 100, orderBy: {field:
|
|
1465
|
+
discussions(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) {
|
|
1466
|
+
pageInfo { hasNextPage endCursor }
|
|
1406
1467
|
nodes {
|
|
1407
1468
|
id
|
|
1408
1469
|
number
|
|
@@ -1424,19 +1485,21 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1424
1485
|
const response = await this.requestGraphQL<{
|
|
1425
1486
|
data?: {
|
|
1426
1487
|
repository?: {
|
|
1427
|
-
discussions?: { nodes?: GraphQLDiscussionNode[] };
|
|
1488
|
+
discussions?: { nodes?: GraphQLDiscussionNode[]; pageInfo?: GitHubPageInfo };
|
|
1428
1489
|
};
|
|
1429
1490
|
};
|
|
1430
1491
|
}>({
|
|
1431
1492
|
token,
|
|
1432
1493
|
query,
|
|
1433
|
-
variables: { owner: repo.owner, repo: repo.repo },
|
|
1494
|
+
variables: { owner: repo.owner, repo: repo.repo, cursor: progress.cursor },
|
|
1434
1495
|
});
|
|
1435
1496
|
|
|
1436
|
-
const
|
|
1437
|
-
const
|
|
1497
|
+
const connection = response.data?.repository?.discussions;
|
|
1498
|
+
const cursor = nextGitHubCursor(connection?.pageInfo, progress.cursor);
|
|
1499
|
+
const discussions = connection?.nodes ?? [];
|
|
1500
|
+
const since = new Date(progress.since).getTime();
|
|
1438
1501
|
|
|
1439
|
-
|
|
1502
|
+
const events = discussions
|
|
1440
1503
|
.map((discussion): EventEnvelope | null => {
|
|
1441
1504
|
const createdAt = new Date(discussion.createdAt);
|
|
1442
1505
|
const updatedAt = new Date(discussion.updatedAt).getTime();
|
|
@@ -1464,20 +1527,23 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1464
1527
|
};
|
|
1465
1528
|
})
|
|
1466
1529
|
.filter((value): value is EventEnvelope => value !== null);
|
|
1530
|
+
return { events, next: cursor ? { cursor } : undefined };
|
|
1467
1531
|
}
|
|
1468
1532
|
|
|
1469
1533
|
private async syncDiscussionComments(
|
|
1470
1534
|
repo: RepoRef,
|
|
1471
|
-
|
|
1535
|
+
progress: GitHubContentProgress,
|
|
1472
1536
|
token: string | null
|
|
1473
|
-
): Promise<
|
|
1537
|
+
): Promise<GitHubPage> {
|
|
1474
1538
|
const query = `
|
|
1475
|
-
query($owner: String!, $repo: String
|
|
1539
|
+
query($owner: String!, $repo: String!, $cursor: String, $commentCursor: String) {
|
|
1476
1540
|
repository(owner: $owner, name: $repo) {
|
|
1477
|
-
discussions(first:
|
|
1541
|
+
discussions(first: 1, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) {
|
|
1542
|
+
pageInfo { hasNextPage endCursor }
|
|
1478
1543
|
nodes {
|
|
1479
1544
|
number
|
|
1480
|
-
comments(first:
|
|
1545
|
+
comments(first: 100, after: $commentCursor) {
|
|
1546
|
+
pageInfo { hasNextPage endCursor }
|
|
1481
1547
|
nodes {
|
|
1482
1548
|
id
|
|
1483
1549
|
body
|
|
@@ -1498,9 +1564,10 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1498
1564
|
data?: {
|
|
1499
1565
|
repository?: {
|
|
1500
1566
|
discussions?: {
|
|
1567
|
+
pageInfo?: GitHubPageInfo;
|
|
1501
1568
|
nodes?: Array<{
|
|
1502
1569
|
number: number;
|
|
1503
|
-
comments?: { nodes?: Array<Omit<GraphQLDiscussionCommentNode, 'discussion'
|
|
1570
|
+
comments?: { nodes?: Array<Omit<GraphQLDiscussionCommentNode, 'discussion'>>; pageInfo?: GitHubPageInfo };
|
|
1504
1571
|
}>;
|
|
1505
1572
|
};
|
|
1506
1573
|
};
|
|
@@ -1508,11 +1575,16 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1508
1575
|
}>({
|
|
1509
1576
|
token,
|
|
1510
1577
|
query,
|
|
1511
|
-
variables: { owner: repo.owner, repo: repo.repo },
|
|
1578
|
+
variables: { owner: repo.owner, repo: repo.repo, cursor: progress.cursor, commentCursor: progress.comment_cursor },
|
|
1512
1579
|
});
|
|
1513
1580
|
|
|
1514
|
-
const
|
|
1515
|
-
const
|
|
1581
|
+
const connection = response.data?.repository?.discussions;
|
|
1582
|
+
const cursor = nextGitHubCursor(connection?.pageInfo, progress.cursor);
|
|
1583
|
+
const discussions = connection?.nodes ?? [];
|
|
1584
|
+
const commentCursor = discussions[0]
|
|
1585
|
+
? nextGitHubCursor(discussions[0].comments?.pageInfo, progress.comment_cursor)
|
|
1586
|
+
: undefined;
|
|
1587
|
+
const since = new Date(progress.since).getTime();
|
|
1516
1588
|
const result: EventEnvelope[] = [];
|
|
1517
1589
|
|
|
1518
1590
|
for (const discussion of discussions) {
|
|
@@ -1543,98 +1615,97 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1543
1615
|
}
|
|
1544
1616
|
}
|
|
1545
1617
|
|
|
1546
|
-
return
|
|
1618
|
+
return {
|
|
1619
|
+
events: result,
|
|
1620
|
+
next: commentCursor ? { cursor: progress.cursor, comment_cursor: commentCursor }
|
|
1621
|
+
: cursor ? { cursor } : undefined,
|
|
1622
|
+
};
|
|
1547
1623
|
}
|
|
1548
1624
|
|
|
1549
1625
|
private async syncStargazers(
|
|
1550
1626
|
repo: RepoRef,
|
|
1551
|
-
checkpoint:
|
|
1627
|
+
checkpoint: GitHubCheckpoint,
|
|
1552
1628
|
token: string | null
|
|
1553
|
-
): Promise<{ events: EventEnvelope[];
|
|
1554
|
-
const previous = this.parseStargazerCheckpoint(checkpoint);
|
|
1629
|
+
): Promise<{ events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] }> {
|
|
1630
|
+
const previous = this.parseStargazerCheckpoint(checkpoint as Record<string, unknown>);
|
|
1555
1631
|
const previousByKey = new Map(previous.map((stargazer) => [stargazer.key, stargazer]));
|
|
1556
|
-
const
|
|
1632
|
+
const scope = JSON.stringify([repo.owner, repo.repo]);
|
|
1633
|
+
const scan = checkpoint.stargazer_scan?.scope === scope ? checkpoint.stargazer_scan : {
|
|
1634
|
+
scope, page: 1, started_at: new Date().toISOString(), current: [],
|
|
1635
|
+
};
|
|
1636
|
+
const currentStargazers = [...scan.current];
|
|
1557
1637
|
const events: EventEnvelope[] = [];
|
|
1558
|
-
const now = new Date();
|
|
1638
|
+
const now = new Date(scan.started_at);
|
|
1559
1639
|
const repoInfo = await this.fetchRepository(repo, token);
|
|
1560
1640
|
const target = this.buildRepoTarget(repo, repoInfo);
|
|
1561
1641
|
let remainingProfileFetches = STARGAZER_PROFILE_FETCH_LIMIT;
|
|
1562
1642
|
|
|
1563
|
-
const
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1643
|
+
const query = new URLSearchParams({ per_page: '100', page: String(scan.page) });
|
|
1644
|
+
const stargazers = await this.requestJson<GitHubStargazerLike[]>({
|
|
1645
|
+
url: `${githubRepoApiUrl(repo, '/stargazers')}?${query.toString()}`,
|
|
1646
|
+
token, accept: 'application/vnd.github.star+json',
|
|
1647
|
+
});
|
|
1648
|
+
if (!Array.isArray(stargazers)) {
|
|
1649
|
+
return { events: [], checkpoint: { ...checkpoint, stargazer_scan: scan }, status: 'more' };
|
|
1650
|
+
}
|
|
1651
|
+
for (const stargazer of stargazers) {
|
|
1652
|
+
const user = stargazer.user ?? stargazer;
|
|
1653
|
+
const login = user.login;
|
|
1654
|
+
if (!login) continue;
|
|
1655
|
+
|
|
1656
|
+
const key = githubUserIdentityKey({ userId: user.id, login });
|
|
1657
|
+
const starredAtIso = toIsoOrUndefined(stargazer.starred_at) ?? now.toISOString();
|
|
1658
|
+
const starredAt = new Date(starredAtIso);
|
|
1659
|
+
if (Number.isNaN(starredAt.getTime())) continue;
|
|
1660
|
+
|
|
1661
|
+
const previousStargazer = previousByKey.get(key);
|
|
1662
|
+
const shouldRefreshProfile = this.shouldRefreshStargazerProfile(previousStargazer, now);
|
|
1663
|
+
const profileFetchedAt = shouldRefreshProfile && remainingProfileFetches > 0
|
|
1664
|
+
? await this.enqueueStargazerProfileEvent({
|
|
1665
|
+
events,
|
|
1666
|
+
login,
|
|
1573
1667
|
token,
|
|
1574
|
-
|
|
1668
|
+
fallback: user,
|
|
1669
|
+
fetchedAt: now,
|
|
1575
1670
|
})
|
|
1576
|
-
);
|
|
1577
|
-
|
|
1578
|
-
},
|
|
1579
|
-
{ pageSize: 100 }
|
|
1580
|
-
);
|
|
1671
|
+
: (previousStargazer?.profile_fetched_at ?? null);
|
|
1672
|
+
if (shouldRefreshProfile && remainingProfileFetches > 0) remainingProfileFetches -= 1;
|
|
1581
1673
|
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
if (Number.isNaN(starredAt.getTime())) continue;
|
|
1592
|
-
|
|
1593
|
-
const previousStargazer = previousByKey.get(key);
|
|
1594
|
-
const shouldRefreshProfile = this.shouldRefreshStargazerProfile(previousStargazer, now);
|
|
1595
|
-
const profileFetchedAt = shouldRefreshProfile && remainingProfileFetches > 0
|
|
1596
|
-
? await this.enqueueStargazerProfileEvent({
|
|
1597
|
-
events,
|
|
1598
|
-
login,
|
|
1599
|
-
token,
|
|
1600
|
-
fallback: user,
|
|
1601
|
-
fetchedAt: now,
|
|
1602
|
-
})
|
|
1603
|
-
: (previousStargazer?.profile_fetched_at ?? null);
|
|
1604
|
-
if (shouldRefreshProfile && remainingProfileFetches > 0) remainingProfileFetches -= 1;
|
|
1605
|
-
|
|
1606
|
-
currentStargazers.push({
|
|
1607
|
-
key,
|
|
1608
|
-
login,
|
|
1609
|
-
starred_at: starredAt.toISOString(),
|
|
1610
|
-
user_id: user.id ?? previousStargazer?.user_id ?? null,
|
|
1611
|
-
user_type: user.type ?? previousStargazer?.user_type ?? null,
|
|
1612
|
-
html_url: user.html_url ?? previousStargazer?.html_url ?? `https://github.com/${login}`,
|
|
1613
|
-
profile_fetched_at: profileFetchedAt,
|
|
1614
|
-
});
|
|
1674
|
+
currentStargazers.push({
|
|
1675
|
+
key,
|
|
1676
|
+
login,
|
|
1677
|
+
starred_at: starredAt.toISOString(),
|
|
1678
|
+
user_id: user.id ?? previousStargazer?.user_id ?? null,
|
|
1679
|
+
user_type: user.type ?? previousStargazer?.user_type ?? null,
|
|
1680
|
+
html_url: user.html_url ?? previousStargazer?.html_url ?? `https://github.com/${login}`,
|
|
1681
|
+
profile_fetched_at: profileFetchedAt,
|
|
1682
|
+
});
|
|
1615
1683
|
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
}
|
|
1684
|
+
if (!previousStargazer) {
|
|
1685
|
+
events.push({
|
|
1686
|
+
origin_id: `stargazer_${repo.owner}_${repo.repo}_${githubKeyForOriginId(key)}`,
|
|
1687
|
+
title: `${login} starred ${repo.owner}/${repo.repo}`,
|
|
1688
|
+
payload_text: `${login} starred ${repo.owner}/${repo.repo}.`,
|
|
1689
|
+
author_name: login,
|
|
1690
|
+
source_url: user.html_url ?? `https://github.com/${login}`,
|
|
1691
|
+
occurred_at: starredAt,
|
|
1692
|
+
origin_type: 'stargazer',
|
|
1693
|
+
score: 1,
|
|
1694
|
+
metadata: {
|
|
1695
|
+
actor: this.buildActor(user, login),
|
|
1696
|
+
target,
|
|
1697
|
+
action: 'starred',
|
|
1698
|
+
starred_at: starredAt.toISOString(),
|
|
1699
|
+
source: 'github_stargazers_snapshot',
|
|
1700
|
+
...this.buildAuthorMetadata({ login, id: user.id }),
|
|
1701
|
+
},
|
|
1702
|
+
});
|
|
1636
1703
|
}
|
|
1637
1704
|
}
|
|
1705
|
+
const current = [...new Map(currentStargazers.map(star => [star.key, star])).values()];
|
|
1706
|
+
if (stargazers.length === 100) {
|
|
1707
|
+
return { events, checkpoint: { ...checkpoint, stargazer_scan: { ...scan, page: scan.page + 1, current } }, status: 'more' };
|
|
1708
|
+
}
|
|
1638
1709
|
|
|
1639
1710
|
const currentKeys = new Set(currentStargazers.map((stargazer) => stargazer.key));
|
|
1640
1711
|
for (const previousStargazer of previous) {
|
|
@@ -1661,7 +1732,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1661
1732
|
});
|
|
1662
1733
|
}
|
|
1663
1734
|
|
|
1664
|
-
return { events,
|
|
1735
|
+
return { events, checkpoint: { last_sync_at: scan.started_at, stargazers: current }, status: 'complete' };
|
|
1665
1736
|
}
|
|
1666
1737
|
|
|
1667
1738
|
private shouldRefreshStargazerProfile(
|
|
@@ -2083,7 +2154,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2083
2154
|
query: string;
|
|
2084
2155
|
variables?: Record<string, unknown>;
|
|
2085
2156
|
}): Promise<T> {
|
|
2086
|
-
|
|
2157
|
+
const response = await this.requestJson<T & { errors?: Array<{ message: string }> }>({
|
|
2087
2158
|
method: 'POST',
|
|
2088
2159
|
url: 'https://api.github.com/graphql',
|
|
2089
2160
|
token: params.token,
|
|
@@ -2095,6 +2166,10 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2095
2166
|
// on a 5xx even though it goes out as POST. A mutation must not set this.
|
|
2096
2167
|
idempotent: true,
|
|
2097
2168
|
});
|
|
2169
|
+
if (response?.errors?.length) {
|
|
2170
|
+
throw new Error(`GitHub GraphQL error: ${response.errors.map(error => error.message).join('; ')}`);
|
|
2171
|
+
}
|
|
2172
|
+
return response;
|
|
2098
2173
|
}
|
|
2099
2174
|
|
|
2100
2175
|
private readonly http = createHttpClient({ errorPrefix: 'GitHub API' });
|