@lobu/runtime-device 20.2.0 → 21.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +44 -38
  3. package/npm-shrinkwrap.json +277 -231
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -79
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +41 -29
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +33 -47
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -3
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -13
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -7
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -29
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +211 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/commands/memory/_lib/schema.d.ts +0 -1
  31. package/vendor/cli/dist/commands/memory/_lib/schema.js +0 -9
  32. package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +0 -2
  33. package/vendor/cli/dist/config/define.d.ts +25 -8
  34. package/vendor/cli/dist/config/define.js +24 -0
  35. package/vendor/cli/dist/connectors/README.md +15 -16
  36. package/vendor/cli/dist/connectors/github.ts +274 -193
  37. package/vendor/cli/dist/connectors/google_calendar.ts +54 -160
  38. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  39. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  40. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  41. package/vendor/cli/dist/connectors/jira.ts +71 -27
  42. package/vendor/cli/dist/connectors/linear.ts +55 -21
  43. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  44. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  45. package/vendor/cli/dist/connectors/producthunt.ts +11 -7
  46. package/vendor/cli/dist/connectors/reddit.ts +36 -6
  47. package/vendor/cli/dist/connectors/rss.ts +2 -2
  48. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  49. package/vendor/cli/dist/connectors/x.ts +32 -17
  50. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  51. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  52. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  53. package/vendor/cli/dist/providers.json +0 -1
  54. package/vendor/cli/dist/runtime-components.json +4 -4
  55. package/vendor/cli/package.json +2 -1
  56. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  57. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  58. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -15
  59. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  60. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  61. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  62. package/vendor/connector-sdk/dist/http-client.js +15 -13
  63. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  64. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  65. package/vendor/connector-sdk/dist/retry.js +40 -64
  66. package/vendor/connector-sdk/package.json +1 -1
  67. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  68. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  69. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  70. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  71. package/vendor/connector-worker/dist/compile/index.js +41 -8
  72. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  73. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  74. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  75. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  76. package/vendor/connector-worker/dist/daemon/client.d.ts +4 -1
  77. package/vendor/connector-worker/dist/daemon/client.js +3 -0
  78. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  79. package/vendor/connector-worker/dist/daemon/executor.js +46 -51
  80. package/vendor/connector-worker/dist/daemon/start.js +6 -2
  81. package/vendor/connector-worker/dist/executor/interface.d.ts +19 -9
  82. package/vendor/connector-worker/dist/executor/isolate.js +85 -28
  83. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  84. package/vendor/connector-worker/package.json +1 -1
  85. package/vendor/core/dist/agent-policy.js +3 -0
  86. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  87. package/vendor/core/dist/connector-query-errors.js +8 -3
  88. package/vendor/core/dist/contracts/tools/manage-auth-profiles.d.ts +14 -0
  89. package/vendor/core/dist/contracts/tools/manage-auth-profiles.js +12 -1
  90. package/vendor/core/dist/contracts/tools/manage-automations.d.ts +0 -10
  91. package/vendor/core/dist/contracts/tools/manage-automations.js +0 -33
  92. package/vendor/core/dist/contracts/tools/manage-classifiers.d.ts +4 -34
  93. package/vendor/core/dist/contracts/tools/manage-classifiers.js +13 -44
  94. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  95. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  96. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  97. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  98. package/vendor/core/dist/contracts/tools/manage-entity.d.ts +0 -9
  99. package/vendor/core/dist/contracts/tools/manage-entity.js +0 -6
  100. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  101. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  102. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  103. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  104. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  105. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  106. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  107. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  108. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  109. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  110. package/vendor/core/dist/contracts/tools/view-attach.d.ts +20 -0
  111. package/vendor/core/dist/contracts/tools/view-attach.js +45 -0
  112. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  113. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  114. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  115. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  116. package/vendor/core/dist/contracts/worker/protocol.d.ts +90 -8
  117. package/vendor/core/dist/contracts/worker/protocol.js +70 -10
  118. package/vendor/core/dist/index.d.ts +1 -1
  119. package/vendor/core/dist/provider-config-types.d.ts +0 -9
  120. package/vendor/core/dist/reserved.d.ts +3 -3
  121. package/vendor/core/dist/reserved.js +9 -1
  122. package/vendor/core/dist/sdk-compat.d.ts +1 -1
  123. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  124. package/vendor/core/package.json +30 -5
  125. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  126. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -17,7 +17,6 @@ import {
17
17
  createHttpClient,
18
18
  type EventAttributionRule,
19
19
  type EventEnvelope,
20
- paginateByOffset,
21
20
  SubscriptionCandidateSchema,
22
21
  type SyncContext,
23
22
  type SyncResult,
@@ -75,9 +74,50 @@ interface GitHubConfig {
75
74
  installation_ref?: number | string;
76
75
  }
77
76
 
77
+ interface GitHubPosition {
78
+ page?: number;
79
+ cursor?: string;
80
+ comment_cursor?: string;
81
+ }
82
+
83
+ interface GitHubContentProgress extends GitHubPosition {
84
+ scope: string;
85
+ since: string;
86
+ started_at: string;
87
+ }
88
+
89
+ interface GitHubPage {
90
+ events: EventEnvelope[];
91
+ next?: GitHubPosition;
92
+ }
93
+
94
+ interface GitHubPageInfo {
95
+ hasNextPage: boolean;
96
+ endCursor?: string | null;
97
+ }
98
+
99
+ function nextGitHubCursor(info: GitHubPageInfo | undefined, current?: string): string | undefined {
100
+ if (!info || typeof info.hasNextPage !== 'boolean') {
101
+ throw new Error('GitHub omitted pagination information.');
102
+ }
103
+ if (!info.hasNextPage) return undefined;
104
+ if (!info.endCursor || info.endCursor === current) {
105
+ throw new Error('GitHub returned a missing or repeated page cursor.');
106
+ }
107
+ return info.endCursor;
108
+ }
109
+
78
110
  interface GitHubCheckpoint {
79
111
  last_sync_at?: string;
112
+ commits?: {
113
+ since: string;
114
+ started_at: string;
115
+ head_sha?: string;
116
+ next_page: number;
117
+ };
118
+ content?: GitHubContentProgress;
80
119
  stargazers?: GitHubStargazerCheckpoint[];
120
+ stargazer_scan?: { scope: string; page: number; started_at: string; current: GitHubStargazerCheckpoint[] };
81
121
  }
82
122
 
83
123
  interface GitHubStargazerCheckpoint {
@@ -243,18 +283,6 @@ function asString(value: unknown): string | undefined {
243
283
  return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
244
284
  }
245
285
 
246
- /**
247
- * Coerce a GitHub list response to an array. `http.json()` returns
248
- * `await response.json()`, which is `null`/`undefined` for an empty-body 200
249
- * (GitHub occasionally serves these). The list sync paths immediately read
250
- * `.length` / iterate, so a bare null threw
251
- * `Cannot read properties of undefined (reading 'length')` and crashed the
252
- * whole sync (#2033). Treat any non-array response as an empty page.
253
- */
254
- function asArray<T>(value: T[] | null | undefined): T[] {
255
- return Array.isArray(value) ? value : [];
256
- }
257
-
258
286
  function toIsoOrUndefined(value: unknown): string | undefined {
259
287
  const str = asString(value);
260
288
  if (!str) return undefined;
@@ -280,9 +308,7 @@ const LOOKBACK_PROP = {
280
308
  const STARGAZER_PROFILE_REFRESH_MS = 30 * 24 * 60 * 60 * 1000;
281
309
  const STARGAZER_PROFILE_FETCH_LIMIT = 25;
282
310
 
283
- // Bound a single commits sync to 30 pages (3000 commits). Incremental `since`
284
- // keeps steady-state runs tiny; the cap only matters on a cold backfill of a
285
- // busy repo, where the remaining history rides subsequent runs.
311
+ // Resume after at most 30 data pages, keeping the same immutable commit graph.
286
312
  const COMMITS_MAX_PAGES = 30;
287
313
 
288
314
  const LABELS_PROP = {
@@ -902,36 +928,37 @@ export default class GitHubConnector extends ConnectorRuntime {
902
928
  );
903
929
  const sinceIso = this.resolveSince(ctx.checkpoint, config.lookback_days ?? 365);
904
930
 
931
+ const checkpoint = (ctx.checkpoint ?? {}) as GitHubCheckpoint;
932
+ let result: { events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] };
905
933
  if (contentType === 'stargazers') {
906
- const result = await this.syncStargazers(repo, ctx.checkpoint, token);
907
- this.stampRepoAttribution(result.events, repo, { attachAutomationSignals });
908
- return {
909
- events: result.events,
910
- checkpoint: {
911
- last_sync_at: new Date().toISOString(),
912
- stargazers: result.currentStargazers,
913
- } as Record<string, unknown>,
914
- metadata: {
915
- items_found: result.events.length,
916
- current_stargazers: result.currentStargazers.length,
917
- },
934
+ result = await this.syncStargazers(repo, checkpoint, token);
935
+ } else if (contentType === 'commits') {
936
+ result = await this.syncCommits(repo, sinceIso, ctx.checkpoint, token);
937
+ } else {
938
+ const labelsFilter = config.labels_filter ?? [];
939
+ const scope = JSON.stringify([repo.owner, repo.repo, contentType, labelsFilter, config.lookback_days ?? 365]);
940
+ const progress = checkpoint.content?.scope === scope ? checkpoint.content : {
941
+ scope,
942
+ since: sinceIso,
943
+ // The REST `since` boundary is exclusive and timestamps have second
944
+ // precision. Overlap that second to recover updates made during a scan.
945
+ started_at: new Date(Math.floor(Date.now() / 1000) * 1000 - 1000).toISOString(),
946
+ };
947
+ const page = await this.syncContent({ repo, contentType, progress, labelsFilter, token });
948
+ result = {
949
+ events: page.events,
950
+ checkpoint: page.next
951
+ ? { ...checkpoint, content: { scope, since: progress.since, started_at: progress.started_at, ...page.next } }
952
+ : { last_sync_at: progress.started_at },
953
+ status: page.next ? 'more' : 'complete',
918
954
  };
919
955
  }
920
-
921
- const events = await this.syncContent({
922
- repo,
923
- contentType,
924
- sinceIso,
925
- labelsFilter: config.labels_filter ?? [],
926
- token,
927
- });
956
+ const { events } = result;
928
957
  this.stampRepoAttribution(events, repo, { attachAutomationSignals });
929
958
 
959
+ await ctx.commit(events, result.checkpoint as Record<string, unknown>);
930
960
  return {
931
- events,
932
- checkpoint: {
933
- last_sync_at: new Date().toISOString(),
934
- } as Record<string, unknown>,
961
+ status: result.status,
935
962
  metadata: {
936
963
  items_found: events.length,
937
964
  },
@@ -1156,58 +1183,83 @@ export default class GitHubConnector extends ConnectorRuntime {
1156
1183
 
1157
1184
  private async syncContent(params: {
1158
1185
  repo: RepoRef;
1159
- contentType: Exclude<GitHubContentType, 'stargazers'>;
1160
- sinceIso: string;
1186
+ contentType: Exclude<GitHubContentType, 'stargazers' | 'commits'>;
1187
+ progress: GitHubContentProgress;
1161
1188
  labelsFilter: string[];
1162
1189
  token: string | null;
1163
- }): Promise<EventEnvelope[]> {
1164
- const { repo, contentType, sinceIso, labelsFilter, token } = params;
1190
+ }): Promise<GitHubPage> {
1191
+ const { repo, contentType, progress, labelsFilter, token } = params;
1165
1192
 
1166
1193
  switch (contentType) {
1167
1194
  case 'issues':
1168
1195
  case 'pull_requests':
1169
- return await this.syncIssuesAndPulls(repo, contentType, sinceIso, labelsFilter, token);
1196
+ return await this.syncIssuesAndPulls(repo, contentType, progress, labelsFilter, token);
1170
1197
  case 'issue_comments':
1171
- return await this.syncIssueComments(repo, sinceIso, token);
1198
+ return await this.syncIssueComments(repo, progress, token);
1172
1199
  case 'pr_comments':
1173
- return await this.syncPullRequestComments(repo, sinceIso, token);
1200
+ return await this.syncPullRequestComments(repo, progress, token);
1174
1201
  case 'discussions':
1175
- return await this.syncDiscussions(repo, sinceIso, token);
1202
+ return await this.syncDiscussions(repo, progress, token);
1176
1203
  case 'discussion_comments':
1177
- return await this.syncDiscussionComments(repo, sinceIso, token);
1178
- case 'commits':
1179
- return await this.syncCommits(repo, sinceIso, token);
1204
+ return await this.syncDiscussionComments(repo, progress, token);
1180
1205
  }
1181
1206
  }
1182
1207
 
1183
1208
  private async syncCommits(
1184
1209
  repo: RepoRef,
1185
1210
  sinceIso: string,
1211
+ rawCheckpoint: Record<string, unknown> | null,
1186
1212
  token: string | null
1187
- ): Promise<EventEnvelope[]> {
1213
+ ): Promise<{ events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] }> {
1188
1214
  const events: EventEnvelope[] = [];
1189
-
1190
- // Commits are durable and fully queryable, so polling recovers the complete
1191
- // "who committed when" history — no webhook dependency. `since` filters by
1192
- // commit date for incremental syncs; pagination is bounded so one run can't
1193
- // walk an unbounded history (deeper backfill rides subsequent runs).
1194
- const pages = paginateByOffset<GitHubCommitLike>(
1195
- async (offset, pageSize) => {
1196
- const query = new URLSearchParams({
1197
- per_page: String(pageSize),
1198
- page: String(offset / pageSize + 1),
1215
+ const checkpoint = (rawCheckpoint ?? {}) as GitHubCheckpoint;
1216
+ const progress: NonNullable<GitHubCheckpoint['commits']> = checkpoint.commits
1217
+ ? { ...checkpoint.commits }
1218
+ : {
1199
1219
  since: sinceIso,
1200
- });
1201
- const url = `${githubRepoApiUrl(repo, '/commits')}?${query.toString()}`;
1202
- const commits = asArray(
1203
- await this.requestJson<GitHubCommitLike[]>({ url, token })
1204
- );
1205
- return { items: commits, hasMore: commits.length === pageSize };
1206
- },
1207
- { pageSize: 100, maxPages: COMMITS_MAX_PAGES }
1208
- );
1220
+ // Use the start, not the end of a potentially multi-run scan. Round down
1221
+ // to GitHub's timestamp precision so commits in this second are replayed.
1222
+ started_at: new Date(Math.floor(Date.now() / 1000) * 1000).toISOString(),
1223
+ next_page: 1,
1224
+ };
1225
+ const more = () => ({
1226
+ events,
1227
+ checkpoint: { ...checkpoint, commits: progress },
1228
+ status: 'more' as const,
1229
+ });
1230
+ const complete = () => ({
1231
+ events,
1232
+ checkpoint: { last_sync_at: progress.started_at },
1233
+ status: 'complete' as const,
1234
+ });
1235
+
1236
+ if (!progress.head_sha) {
1237
+ // Resolve the unfiltered default-branch head before paging. A date-filtered
1238
+ // first commit need not be the tip; pinning that could omit merge ancestry.
1239
+ const head = await this.requestJson<GitHubCommitLike[]>({
1240
+ url: `${githubRepoApiUrl(repo, '/commits')}?per_page=1`,
1241
+ token,
1242
+ });
1243
+ if (!Array.isArray(head)) return more();
1244
+ if (head.length === 0) return complete();
1245
+ progress.head_sha = head[0]?.sha;
1246
+ if (!progress.head_sha) return more();
1247
+ }
1209
1248
 
1210
- for await (const commits of pages) {
1249
+ for (let page = 0; page < COMMITS_MAX_PAGES; page++) {
1250
+ const query = new URLSearchParams({
1251
+ per_page: '100',
1252
+ page: String(progress.next_page),
1253
+ since: progress.since,
1254
+ // New commits and force pushes cannot move offsets in this snapshot.
1255
+ sha: progress.head_sha,
1256
+ });
1257
+ const commits = await this.requestJson<GitHubCommitLike[]>({
1258
+ url: `${githubRepoApiUrl(repo, '/commits')}?${query.toString()}`,
1259
+ token,
1260
+ });
1261
+ // An empty body is indeterminate, not evidence that history is exhausted.
1262
+ if (!Array.isArray(commits)) return more();
1211
1263
  for (const commit of commits) {
1212
1264
  if (!commit.sha) continue;
1213
1265
  const authoredAtIso = commit.commit?.author?.date;
@@ -1241,31 +1293,36 @@ export default class GitHubConnector extends ConnectorRuntime {
1241
1293
  },
1242
1294
  });
1243
1295
  }
1296
+ progress.next_page += 1;
1297
+ if (commits.length < 100) return complete();
1244
1298
  }
1245
1299
 
1246
- return events;
1300
+ return more();
1247
1301
  }
1248
1302
 
1249
1303
  private async syncIssuesAndPulls(
1250
1304
  repo: RepoRef,
1251
1305
  contentType: 'issues' | 'pull_requests',
1252
- sinceIso: string,
1306
+ progress: GitHubContentProgress,
1253
1307
  labelsFilter: string[],
1254
1308
  token: string | null
1255
- ): Promise<EventEnvelope[]> {
1309
+ ): Promise<GitHubPage> {
1256
1310
  const query = new URLSearchParams({
1257
1311
  state: 'all',
1258
1312
  per_page: '100',
1259
- sort: 'updated',
1260
- direction: 'desc',
1261
- since: sinceIso,
1313
+ // Updates must not move consumed records to a different offset.
1314
+ sort: 'created',
1315
+ direction: 'asc',
1316
+ since: progress.since,
1317
+ page: String(progress.page ?? 1),
1262
1318
  });
1263
1319
  if (labelsFilter.length > 0) {
1264
1320
  query.set('labels', labelsFilter.join(','));
1265
1321
  }
1266
1322
 
1267
1323
  const url = `${githubRepoApiUrl(repo, '/issues')}?${query.toString()}`;
1268
- const items = asArray(await this.requestJson<GitHubIssueLike[]>({ url, token }));
1324
+ const items = await this.requestJson<GitHubIssueLike[]>({ url, token });
1325
+ if (!Array.isArray(items)) return { events: [], next: progress };
1269
1326
  const events: EventEnvelope[] = [];
1270
1327
 
1271
1328
  for (const item of items) {
@@ -1302,26 +1359,27 @@ export default class GitHubConnector extends ConnectorRuntime {
1302
1359
  });
1303
1360
  }
1304
1361
 
1305
- return events;
1362
+ return { events, next: items.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
1306
1363
  }
1307
1364
 
1308
1365
  private async syncIssueComments(
1309
1366
  repo: RepoRef,
1310
- sinceIso: string,
1367
+ progress: GitHubContentProgress,
1311
1368
  token: string | null
1312
- ): Promise<EventEnvelope[]> {
1369
+ ): Promise<GitHubPage> {
1313
1370
  const query = new URLSearchParams({
1314
1371
  per_page: '100',
1315
- sort: 'updated',
1316
- direction: 'desc',
1317
- since: sinceIso,
1372
+ // Updates must not move consumed records to a different offset.
1373
+ sort: 'created',
1374
+ direction: 'asc',
1375
+ since: progress.since,
1376
+ page: String(progress.page ?? 1),
1318
1377
  });
1319
1378
  const url = `${githubRepoApiUrl(repo, '/issues/comments')}?${query.toString()}`;
1320
- const comments = asArray(
1321
- await this.requestJson<GitHubCommentLike[]>({ url, token })
1322
- );
1379
+ const comments = await this.requestJson<GitHubCommentLike[]>({ url, token });
1380
+ if (!Array.isArray(comments)) return { events: [], next: progress };
1323
1381
 
1324
- return comments
1382
+ const events = comments
1325
1383
  .map((comment): EventEnvelope | null => {
1326
1384
  const createdAt = new Date(comment.created_at);
1327
1385
  if (Number.isNaN(createdAt.getTime())) return null;
@@ -1347,25 +1405,27 @@ export default class GitHubConnector extends ConnectorRuntime {
1347
1405
  };
1348
1406
  })
1349
1407
  .filter((value): value is EventEnvelope => value !== null);
1408
+ return { events, next: comments.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
1350
1409
  }
1351
1410
 
1352
1411
  private async syncPullRequestComments(
1353
1412
  repo: RepoRef,
1354
- sinceIso: string,
1413
+ progress: GitHubContentProgress,
1355
1414
  token: string | null
1356
- ): Promise<EventEnvelope[]> {
1415
+ ): Promise<GitHubPage> {
1357
1416
  const query = new URLSearchParams({
1358
1417
  per_page: '100',
1359
- sort: 'updated',
1360
- direction: 'desc',
1361
- since: sinceIso,
1418
+ // Updates must not move consumed records to a different offset.
1419
+ sort: 'created',
1420
+ direction: 'asc',
1421
+ since: progress.since,
1422
+ page: String(progress.page ?? 1),
1362
1423
  });
1363
1424
  const url = `${githubRepoApiUrl(repo, '/pulls/comments')}?${query.toString()}`;
1364
- const comments = asArray(
1365
- await this.requestJson<GitHubCommentLike[]>({ url, token })
1366
- );
1425
+ const comments = await this.requestJson<GitHubCommentLike[]>({ url, token });
1426
+ if (!Array.isArray(comments)) return { events: [], next: progress };
1367
1427
 
1368
- return comments
1428
+ const events = comments
1369
1429
  .map((comment): EventEnvelope | null => {
1370
1430
  const createdAt = new Date(comment.created_at);
1371
1431
  if (Number.isNaN(createdAt.getTime())) return null;
@@ -1391,17 +1451,19 @@ export default class GitHubConnector extends ConnectorRuntime {
1391
1451
  };
1392
1452
  })
1393
1453
  .filter((value): value is EventEnvelope => value !== null);
1454
+ return { events, next: comments.length === 100 ? { page: (progress.page ?? 1) + 1 } : undefined };
1394
1455
  }
1395
1456
 
1396
1457
  private async syncDiscussions(
1397
1458
  repo: RepoRef,
1398
- sinceIso: string,
1459
+ progress: GitHubContentProgress,
1399
1460
  token: string | null
1400
- ): Promise<EventEnvelope[]> {
1461
+ ): Promise<GitHubPage> {
1401
1462
  const query = `
1402
- query($owner: String!, $repo: String!) {
1463
+ query($owner: String!, $repo: String!, $cursor: String) {
1403
1464
  repository(owner: $owner, name: $repo) {
1404
- discussions(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}) {
1465
+ discussions(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) {
1466
+ pageInfo { hasNextPage endCursor }
1405
1467
  nodes {
1406
1468
  id
1407
1469
  number
@@ -1423,19 +1485,21 @@ export default class GitHubConnector extends ConnectorRuntime {
1423
1485
  const response = await this.requestGraphQL<{
1424
1486
  data?: {
1425
1487
  repository?: {
1426
- discussions?: { nodes?: GraphQLDiscussionNode[] };
1488
+ discussions?: { nodes?: GraphQLDiscussionNode[]; pageInfo?: GitHubPageInfo };
1427
1489
  };
1428
1490
  };
1429
1491
  }>({
1430
1492
  token,
1431
1493
  query,
1432
- variables: { owner: repo.owner, repo: repo.repo },
1494
+ variables: { owner: repo.owner, repo: repo.repo, cursor: progress.cursor },
1433
1495
  });
1434
1496
 
1435
- const discussions = response.data?.repository?.discussions?.nodes ?? [];
1436
- const since = new Date(sinceIso).getTime();
1497
+ const connection = response.data?.repository?.discussions;
1498
+ const cursor = nextGitHubCursor(connection?.pageInfo, progress.cursor);
1499
+ const discussions = connection?.nodes ?? [];
1500
+ const since = new Date(progress.since).getTime();
1437
1501
 
1438
- return discussions
1502
+ const events = discussions
1439
1503
  .map((discussion): EventEnvelope | null => {
1440
1504
  const createdAt = new Date(discussion.createdAt);
1441
1505
  const updatedAt = new Date(discussion.updatedAt).getTime();
@@ -1463,20 +1527,23 @@ export default class GitHubConnector extends ConnectorRuntime {
1463
1527
  };
1464
1528
  })
1465
1529
  .filter((value): value is EventEnvelope => value !== null);
1530
+ return { events, next: cursor ? { cursor } : undefined };
1466
1531
  }
1467
1532
 
1468
1533
  private async syncDiscussionComments(
1469
1534
  repo: RepoRef,
1470
- sinceIso: string,
1535
+ progress: GitHubContentProgress,
1471
1536
  token: string | null
1472
- ): Promise<EventEnvelope[]> {
1537
+ ): Promise<GitHubPage> {
1473
1538
  const query = `
1474
- query($owner: String!, $repo: String!) {
1539
+ query($owner: String!, $repo: String!, $cursor: String, $commentCursor: String) {
1475
1540
  repository(owner: $owner, name: $repo) {
1476
- discussions(first: 50, orderBy: {field: UPDATED_AT, direction: DESC}) {
1541
+ discussions(first: 1, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) {
1542
+ pageInfo { hasNextPage endCursor }
1477
1543
  nodes {
1478
1544
  number
1479
- comments(first: 50) {
1545
+ comments(first: 100, after: $commentCursor) {
1546
+ pageInfo { hasNextPage endCursor }
1480
1547
  nodes {
1481
1548
  id
1482
1549
  body
@@ -1497,9 +1564,10 @@ export default class GitHubConnector extends ConnectorRuntime {
1497
1564
  data?: {
1498
1565
  repository?: {
1499
1566
  discussions?: {
1567
+ pageInfo?: GitHubPageInfo;
1500
1568
  nodes?: Array<{
1501
1569
  number: number;
1502
- comments?: { nodes?: Array<Omit<GraphQLDiscussionCommentNode, 'discussion'>> };
1570
+ comments?: { nodes?: Array<Omit<GraphQLDiscussionCommentNode, 'discussion'>>; pageInfo?: GitHubPageInfo };
1503
1571
  }>;
1504
1572
  };
1505
1573
  };
@@ -1507,11 +1575,16 @@ export default class GitHubConnector extends ConnectorRuntime {
1507
1575
  }>({
1508
1576
  token,
1509
1577
  query,
1510
- variables: { owner: repo.owner, repo: repo.repo },
1578
+ variables: { owner: repo.owner, repo: repo.repo, cursor: progress.cursor, commentCursor: progress.comment_cursor },
1511
1579
  });
1512
1580
 
1513
- const discussions = response.data?.repository?.discussions?.nodes ?? [];
1514
- const since = new Date(sinceIso).getTime();
1581
+ const connection = response.data?.repository?.discussions;
1582
+ const cursor = nextGitHubCursor(connection?.pageInfo, progress.cursor);
1583
+ const discussions = connection?.nodes ?? [];
1584
+ const commentCursor = discussions[0]
1585
+ ? nextGitHubCursor(discussions[0].comments?.pageInfo, progress.comment_cursor)
1586
+ : undefined;
1587
+ const since = new Date(progress.since).getTime();
1515
1588
  const result: EventEnvelope[] = [];
1516
1589
 
1517
1590
  for (const discussion of discussions) {
@@ -1542,98 +1615,97 @@ export default class GitHubConnector extends ConnectorRuntime {
1542
1615
  }
1543
1616
  }
1544
1617
 
1545
- return result;
1618
+ return {
1619
+ events: result,
1620
+ next: commentCursor ? { cursor: progress.cursor, comment_cursor: commentCursor }
1621
+ : cursor ? { cursor } : undefined,
1622
+ };
1546
1623
  }
1547
1624
 
1548
1625
  private async syncStargazers(
1549
1626
  repo: RepoRef,
1550
- checkpoint: Record<string, unknown> | null,
1627
+ checkpoint: GitHubCheckpoint,
1551
1628
  token: string | null
1552
- ): Promise<{ events: EventEnvelope[]; currentStargazers: GitHubStargazerCheckpoint[] }> {
1553
- const previous = this.parseStargazerCheckpoint(checkpoint);
1629
+ ): Promise<{ events: EventEnvelope[]; checkpoint: GitHubCheckpoint; status: SyncResult['status'] }> {
1630
+ const previous = this.parseStargazerCheckpoint(checkpoint as Record<string, unknown>);
1554
1631
  const previousByKey = new Map(previous.map((stargazer) => [stargazer.key, stargazer]));
1555
- const currentStargazers: GitHubStargazerCheckpoint[] = [];
1632
+ const scope = JSON.stringify([repo.owner, repo.repo]);
1633
+ const scan = checkpoint.stargazer_scan?.scope === scope ? checkpoint.stargazer_scan : {
1634
+ scope, page: 1, started_at: new Date().toISOString(), current: [],
1635
+ };
1636
+ const currentStargazers = [...scan.current];
1556
1637
  const events: EventEnvelope[] = [];
1557
- const now = new Date();
1638
+ const now = new Date(scan.started_at);
1558
1639
  const repoInfo = await this.fetchRepository(repo, token);
1559
1640
  const target = this.buildRepoTarget(repo, repoInfo);
1560
1641
  let remainingProfileFetches = STARGAZER_PROFILE_FETCH_LIMIT;
1561
1642
 
1562
- const pages = paginateByOffset<GitHubStargazerLike>(
1563
- async (offset, pageSize) => {
1564
- const query = new URLSearchParams({
1565
- per_page: String(pageSize),
1566
- page: String(offset / pageSize + 1),
1567
- });
1568
- const url = `${githubRepoApiUrl(repo, '/stargazers')}?${query.toString()}`;
1569
- const stargazers = asArray(
1570
- await this.requestJson<GitHubStargazerLike[]>({
1571
- url,
1643
+ const query = new URLSearchParams({ per_page: '100', page: String(scan.page) });
1644
+ const stargazers = await this.requestJson<GitHubStargazerLike[]>({
1645
+ url: `${githubRepoApiUrl(repo, '/stargazers')}?${query.toString()}`,
1646
+ token, accept: 'application/vnd.github.star+json',
1647
+ });
1648
+ if (!Array.isArray(stargazers)) {
1649
+ return { events: [], checkpoint: { ...checkpoint, stargazer_scan: scan }, status: 'more' };
1650
+ }
1651
+ for (const stargazer of stargazers) {
1652
+ const user = stargazer.user ?? stargazer;
1653
+ const login = user.login;
1654
+ if (!login) continue;
1655
+
1656
+ const key = githubUserIdentityKey({ userId: user.id, login });
1657
+ const starredAtIso = toIsoOrUndefined(stargazer.starred_at) ?? now.toISOString();
1658
+ const starredAt = new Date(starredAtIso);
1659
+ if (Number.isNaN(starredAt.getTime())) continue;
1660
+
1661
+ const previousStargazer = previousByKey.get(key);
1662
+ const shouldRefreshProfile = this.shouldRefreshStargazerProfile(previousStargazer, now);
1663
+ const profileFetchedAt = shouldRefreshProfile && remainingProfileFetches > 0
1664
+ ? await this.enqueueStargazerProfileEvent({
1665
+ events,
1666
+ login,
1572
1667
  token,
1573
- accept: 'application/vnd.github.star+json',
1668
+ fallback: user,
1669
+ fetchedAt: now,
1574
1670
  })
1575
- );
1576
- return { items: stargazers, hasMore: stargazers.length === pageSize };
1577
- },
1578
- { pageSize: 100 }
1579
- );
1671
+ : (previousStargazer?.profile_fetched_at ?? null);
1672
+ if (shouldRefreshProfile && remainingProfileFetches > 0) remainingProfileFetches -= 1;
1580
1673
 
1581
- for await (const stargazers of pages) {
1582
- for (const stargazer of stargazers) {
1583
- const user = stargazer.user ?? stargazer;
1584
- const login = user.login;
1585
- if (!login) continue;
1586
-
1587
- const key = githubUserIdentityKey({ userId: user.id, login });
1588
- const starredAtIso = toIsoOrUndefined(stargazer.starred_at) ?? now.toISOString();
1589
- const starredAt = new Date(starredAtIso);
1590
- if (Number.isNaN(starredAt.getTime())) continue;
1591
-
1592
- const previousStargazer = previousByKey.get(key);
1593
- const shouldRefreshProfile = this.shouldRefreshStargazerProfile(previousStargazer, now);
1594
- const profileFetchedAt = shouldRefreshProfile && remainingProfileFetches > 0
1595
- ? await this.enqueueStargazerProfileEvent({
1596
- events,
1597
- login,
1598
- token,
1599
- fallback: user,
1600
- fetchedAt: now,
1601
- })
1602
- : (previousStargazer?.profile_fetched_at ?? null);
1603
- if (shouldRefreshProfile && remainingProfileFetches > 0) remainingProfileFetches -= 1;
1604
-
1605
- currentStargazers.push({
1606
- key,
1607
- login,
1608
- starred_at: starredAt.toISOString(),
1609
- user_id: user.id ?? previousStargazer?.user_id ?? null,
1610
- user_type: user.type ?? previousStargazer?.user_type ?? null,
1611
- html_url: user.html_url ?? previousStargazer?.html_url ?? `https://github.com/${login}`,
1612
- profile_fetched_at: profileFetchedAt,
1613
- });
1674
+ currentStargazers.push({
1675
+ key,
1676
+ login,
1677
+ starred_at: starredAt.toISOString(),
1678
+ user_id: user.id ?? previousStargazer?.user_id ?? null,
1679
+ user_type: user.type ?? previousStargazer?.user_type ?? null,
1680
+ html_url: user.html_url ?? previousStargazer?.html_url ?? `https://github.com/${login}`,
1681
+ profile_fetched_at: profileFetchedAt,
1682
+ });
1614
1683
 
1615
- if (!previousStargazer) {
1616
- events.push({
1617
- origin_id: `stargazer_${repo.owner}_${repo.repo}_${githubKeyForOriginId(key)}`,
1618
- title: `${login} starred ${repo.owner}/${repo.repo}`,
1619
- payload_text: `${login} starred ${repo.owner}/${repo.repo}.`,
1620
- author_name: login,
1621
- source_url: user.html_url ?? `https://github.com/${login}`,
1622
- occurred_at: starredAt,
1623
- origin_type: 'stargazer',
1624
- score: 1,
1625
- metadata: {
1626
- actor: this.buildActor(user, login),
1627
- target,
1628
- action: 'starred',
1629
- starred_at: starredAt.toISOString(),
1630
- source: 'github_stargazers_snapshot',
1631
- ...this.buildAuthorMetadata({ login, id: user.id }),
1632
- },
1633
- });
1634
- }
1684
+ if (!previousStargazer) {
1685
+ events.push({
1686
+ origin_id: `stargazer_${repo.owner}_${repo.repo}_${githubKeyForOriginId(key)}`,
1687
+ title: `${login} starred ${repo.owner}/${repo.repo}`,
1688
+ payload_text: `${login} starred ${repo.owner}/${repo.repo}.`,
1689
+ author_name: login,
1690
+ source_url: user.html_url ?? `https://github.com/${login}`,
1691
+ occurred_at: starredAt,
1692
+ origin_type: 'stargazer',
1693
+ score: 1,
1694
+ metadata: {
1695
+ actor: this.buildActor(user, login),
1696
+ target,
1697
+ action: 'starred',
1698
+ starred_at: starredAt.toISOString(),
1699
+ source: 'github_stargazers_snapshot',
1700
+ ...this.buildAuthorMetadata({ login, id: user.id }),
1701
+ },
1702
+ });
1635
1703
  }
1636
1704
  }
1705
+ const current = [...new Map(currentStargazers.map(star => [star.key, star])).values()];
1706
+ if (stargazers.length === 100) {
1707
+ return { events, checkpoint: { ...checkpoint, stargazer_scan: { ...scan, page: scan.page + 1, current } }, status: 'more' };
1708
+ }
1637
1709
 
1638
1710
  const currentKeys = new Set(currentStargazers.map((stargazer) => stargazer.key));
1639
1711
  for (const previousStargazer of previous) {
@@ -1660,7 +1732,7 @@ export default class GitHubConnector extends ConnectorRuntime {
1660
1732
  });
1661
1733
  }
1662
1734
 
1663
- return { events, currentStargazers };
1735
+ return { events, checkpoint: { last_sync_at: scan.started_at, stargazers: current }, status: 'complete' };
1664
1736
  }
1665
1737
 
1666
1738
  private shouldRefreshStargazerProfile(
@@ -2082,7 +2154,7 @@ export default class GitHubConnector extends ConnectorRuntime {
2082
2154
  query: string;
2083
2155
  variables?: Record<string, unknown>;
2084
2156
  }): Promise<T> {
2085
- return await this.requestJson<T>({
2157
+ const response = await this.requestJson<T & { errors?: Array<{ message: string }> }>({
2086
2158
  method: 'POST',
2087
2159
  url: 'https://api.github.com/graphql',
2088
2160
  token: params.token,
@@ -2090,7 +2162,14 @@ export default class GitHubConnector extends ConnectorRuntime {
2090
2162
  query: params.query,
2091
2163
  variables: params.variables ?? {},
2092
2164
  },
2165
+ // Every caller sends a query (discussion reads), which is safe to repeat
2166
+ // on a 5xx even though it goes out as POST. A mutation must not set this.
2167
+ idempotent: true,
2093
2168
  });
2169
+ if (response?.errors?.length) {
2170
+ throw new Error(`GitHub GraphQL error: ${response.errors.map(error => error.message).join('; ')}`);
2171
+ }
2172
+ return response;
2094
2173
  }
2095
2174
 
2096
2175
  private readonly http = createHttpClient({ errorPrefix: 'GitHub API' });
@@ -2101,6 +2180,7 @@ export default class GitHubConnector extends ConnectorRuntime {
2101
2180
  token: string | null;
2102
2181
  body?: Record<string, unknown>;
2103
2182
  accept?: string;
2183
+ idempotent?: boolean;
2104
2184
  }): Promise<T> {
2105
2185
  const headers: Record<string, string> = {
2106
2186
  Accept: params.accept ?? 'application/vnd.github+json',
@@ -2115,6 +2195,7 @@ export default class GitHubConnector extends ConnectorRuntime {
2115
2195
  method: params.method ?? 'GET',
2116
2196
  headers,
2117
2197
  body: params.body ? JSON.stringify(params.body) : undefined,
2198
+ idempotent: params.idempotent,
2118
2199
  });
2119
2200
  }
2120
2201
  }