@lobu/runtime-device 20.1.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +46 -40
  3. package/npm-shrinkwrap.json +283 -237
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/commands/login.js +3 -0
  31. package/vendor/cli/dist/config/define.d.ts +25 -8
  32. package/vendor/cli/dist/config/define.js +24 -0
  33. package/vendor/cli/dist/connectors/README.md +15 -16
  34. package/vendor/cli/dist/connectors/chat-user-identity.ts +41 -0
  35. package/vendor/cli/dist/connectors/github.ts +15 -9
  36. package/vendor/cli/dist/connectors/google-identity.ts +11 -0
  37. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  38. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  39. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  40. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  41. package/vendor/cli/dist/connectors/jira.ts +62 -27
  42. package/vendor/cli/dist/connectors/linear.ts +55 -21
  43. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  44. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  45. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  46. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  47. package/vendor/cli/dist/connectors/rss.ts +2 -2
  48. package/vendor/cli/dist/connectors/slack-identity.ts +19 -0
  49. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  50. package/vendor/cli/dist/connectors/x.ts +32 -17
  51. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  52. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  53. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  54. package/vendor/cli/dist/internal/credentials.js +5 -1
  55. package/vendor/cli/dist/internal/oauth.d.ts +15 -1
  56. package/vendor/cli/dist/internal/oauth.js +9 -1
  57. package/vendor/cli/dist/runtime-components.json +4 -4
  58. package/vendor/cli/package.json +2 -1
  59. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  60. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  61. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  62. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  63. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  64. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  65. package/vendor/connector-sdk/dist/http-client.js +15 -13
  66. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  67. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  68. package/vendor/connector-sdk/dist/retry.js +40 -64
  69. package/vendor/connector-sdk/package.json +1 -1
  70. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  71. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  72. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  73. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  74. package/vendor/connector-worker/dist/compile/index.js +41 -8
  75. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  76. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  77. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  78. package/vendor/connector-worker/dist/daemon/automation-process.d.ts +32 -0
  79. package/vendor/connector-worker/dist/daemon/automation-process.js +102 -1
  80. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  81. package/vendor/connector-worker/dist/daemon/builtins/index.js +8 -0
  82. package/vendor/connector-worker/dist/daemon/builtins/os-shell.d.ts +13 -1
  83. package/vendor/connector-worker/dist/daemon/builtins/os-shell.js +28 -2
  84. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  85. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  86. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  87. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  88. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  89. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  90. package/vendor/connector-worker/package.json +1 -1
  91. package/vendor/core/dist/agent-policy.js +3 -0
  92. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  93. package/vendor/core/dist/connector-query-errors.js +8 -3
  94. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  95. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  96. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  97. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  98. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  99. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  100. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  101. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  102. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  103. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  104. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  105. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  106. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  107. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  108. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  109. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  110. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  111. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  112. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  113. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  114. package/vendor/core/dist/credentials.d.ts +16 -1
  115. package/vendor/core/dist/credentials.js +10 -1
  116. package/vendor/core/dist/reserved.d.ts +3 -3
  117. package/vendor/core/dist/reserved.js +9 -1
  118. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  119. package/vendor/core/package.json +25 -5
  120. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  121. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -18,6 +18,18 @@
18
18
  * these links authorize approvals and agent re-binding.
19
19
  */
20
20
 
21
+ /**
22
+ * How a platform's stored user keys are scoped for a reverse lookup.
23
+ *
24
+ * `team-prefix` — keys for one tenant all begin with `prefix`, so a search must
25
+ * be constrained to it (Slack: `T0XYZ:`).
26
+ * `global` — ids are unique platform-wide, so the namespace needs no narrowing
27
+ * (Google: one account can only ever have one id).
28
+ */
29
+ export type ChatUserKeyScope =
30
+ | { kind: 'team-prefix'; prefix: string }
31
+ | { kind: 'global' };
32
+
21
33
  /** How one chat platform keys a sender identity. */
22
34
  export interface ChatUserIdentity {
23
35
  /** Chat platform key an inbound message arrives on (`slack`, `gchat`, …). */
@@ -42,4 +54,33 @@ export interface ChatUserIdentity {
42
54
  teamId: string | null | undefined,
43
55
  platformUserId: string | null | undefined,
44
56
  ): string | null;
57
+
58
+ /**
59
+ * The REVERSE of `buildUserKey`, for "which platform id does this Lobu user
60
+ * have here?": how stored keys are scoped when searching within `teamId`.
61
+ *
62
+ * Required, not optional, and deliberately NOT a bare nullable string. The
63
+ * two failure shapes must not collapse into one value: `global` means the
64
+ * platform has no tenant axis so the whole namespace is in scope, while null
65
+ * means this platform REQUIRES a team and did not get a usable one. A caller
66
+ * that treats those alike widens a Slack search past its `TEAM:` prefix and
67
+ * can return an id from a DIFFERENT workspace — the exact cross-tenant bleed
68
+ * `buildUserKey` exists to prevent. Callers must refuse on null.
69
+ */
70
+ userKeyScope(teamId: string | null | undefined): ChatUserKeyScope | null;
71
+
72
+ /**
73
+ * Recover from a stored key the id the PLATFORM addresses this user by — the
74
+ * form its API takes (`openDM`, a mention), which is also the form
75
+ * `buildUserKey` accepts back.
76
+ *
77
+ * NOT "the key minus its scope prefix". Google stores the bare account id but
78
+ * addresses `users/<id>`, and the chat SDK routes a DM by inferring the
79
+ * adapter from exactly that prefix, so a bare id does not reach Google Chat
80
+ * at all. Each platform owns the difference; core must never re-derive it.
81
+ *
82
+ * Returns null when the stored key is not one this platform could have
83
+ * written — fail closed rather than address a malformed id.
84
+ */
85
+ platformUserIdFromKey(key: string): string | null;
45
86
  }
@@ -905,12 +905,13 @@ export default class GitHubConnector extends ConnectorRuntime {
905
905
  if (contentType === 'stargazers') {
906
906
  const result = await this.syncStargazers(repo, ctx.checkpoint, token);
907
907
  this.stampRepoAttribution(result.events, repo, { attachAutomationSignals });
908
+ // One commit: removed stargazers are only known against the full snapshot.
909
+ await ctx.commit(result.events, {
910
+ last_sync_at: new Date().toISOString(),
911
+ stargazers: result.currentStargazers,
912
+ } as Record<string, unknown>);
908
913
  return {
909
- events: result.events,
910
- checkpoint: {
911
- last_sync_at: new Date().toISOString(),
912
- stargazers: result.currentStargazers,
913
- } as Record<string, unknown>,
914
+ status: 'complete',
914
915
  metadata: {
915
916
  items_found: result.events.length,
916
917
  current_stargazers: result.currentStargazers.length,
@@ -927,11 +928,11 @@ export default class GitHubConnector extends ConnectorRuntime {
927
928
  });
928
929
  this.stampRepoAttribution(events, repo, { attachAutomationSignals });
929
930
 
931
+ await ctx.commit(events, {
932
+ last_sync_at: new Date().toISOString(),
933
+ } as Record<string, unknown>);
930
934
  return {
931
- events,
932
- checkpoint: {
933
- last_sync_at: new Date().toISOString(),
934
- } as Record<string, unknown>,
935
+ status: 'complete',
935
936
  metadata: {
936
937
  items_found: events.length,
937
938
  },
@@ -2090,6 +2091,9 @@ export default class GitHubConnector extends ConnectorRuntime {
2090
2091
  query: params.query,
2091
2092
  variables: params.variables ?? {},
2092
2093
  },
2094
+ // Every caller sends a query (discussion reads), which is safe to repeat
2095
+ // on a 5xx even though it goes out as POST. A mutation must not set this.
2096
+ idempotent: true,
2093
2097
  });
2094
2098
  }
2095
2099
 
@@ -2101,6 +2105,7 @@ export default class GitHubConnector extends ConnectorRuntime {
2101
2105
  token: string | null;
2102
2106
  body?: Record<string, unknown>;
2103
2107
  accept?: string;
2108
+ idempotent?: boolean;
2104
2109
  }): Promise<T> {
2105
2110
  const headers: Record<string, string> = {
2106
2111
  Accept: params.accept ?? 'application/vnd.github+json',
@@ -2115,6 +2120,7 @@ export default class GitHubConnector extends ConnectorRuntime {
2115
2120
  method: params.method ?? 'GET',
2116
2121
  headers,
2117
2122
  body: params.body ? JSON.stringify(params.body) : undefined,
2123
+ idempotent: params.idempotent,
2118
2124
  });
2119
2125
  }
2120
2126
  }
@@ -104,4 +104,15 @@ export const gchatChatUserIdentity: ChatUserIdentity = {
104
104
  buildUserKey(_teamId, platformUserId) {
105
105
  return normalizeGoogleUserId(platformUserId);
106
106
  },
107
+ // Global for the same reason `buildUserKey` ignores the team: one Google
108
+ // account has exactly one id, so there is no tenant axis to narrow by and
109
+ // nothing to bleed across. Never null — a missing team is not an error here.
110
+ userKeyScope: () => ({ kind: 'global' }),
111
+ // Re-wraps the bare stored id as the resource name Chat addresses people by.
112
+ // The `users/` prefix is load-bearing on the way out as well as in: the chat
113
+ // SDK picks the Google Chat adapter off it, so a bare id is not routable.
114
+ platformUserIdFromKey(key) {
115
+ const bare = normalizeGoogleUserId(key);
116
+ return bare ? `${GCHAT_USER_PREFIX}${bare}` : null;
117
+ },
107
118
  };
@@ -457,7 +457,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
457
457
  const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
458
458
 
459
459
  const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
460
- const events: EventEnvelope[] = [];
461
460
  const durableChanges = ctx.feedKey === 'changes';
462
461
 
463
462
  // A checkpoint minted under a different scope (or by an older version that
@@ -467,10 +466,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
467
466
  const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
468
467
  const resumable = checkpoint.scope === scope;
469
468
 
470
- /** Traversal reached the last page: store its cursor and stamp the run. */
471
- const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
472
- const result = this.buildResult(items, syncToken);
473
- return { ...result, checkpoint: { ...result.checkpoint, scope } };
469
+ /** Traversal reached the last page: commit it with its cursor and stamp the run. */
470
+ const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
471
+ await ctx.commit(items, {
472
+ ...this.completedCheckpoint(syncToken),
473
+ scope,
474
+ } as Record<string, unknown>);
475
+ return { status: 'complete' };
474
476
  };
475
477
 
476
478
  if (resumable && checkpoint.sync_token) {
@@ -517,15 +519,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
517
519
  const baseParams = params.toString();
518
520
 
519
521
  /**
520
- * Ran out of budget mid-traversal: park the next page token and return no
521
- * sync token and no `last_sync_at`, so nothing downstream can read this run
522
- * as a completed window. The next scheduled run resumes from `token`.
522
+ * Mid-traversal: the next page token, with no sync token and no
523
+ * `last_sync_at`, so nothing downstream can read it as a completed window.
524
+ * A run that stops here resumes from `token`.
523
525
  */
524
- const park = (token: string): SyncResult => ({
525
- ...this.buildResult(events, undefined),
526
- checkpoint: { scope, pending: { params: baseParams, page_token: token } },
527
- });
526
+ const parked = (token: string) =>
527
+ ({ scope, pending: { params: baseParams, page_token: token } }) as Record<string, unknown>;
528
528
 
529
+ let collected = 0;
529
530
  let pageToken = pending?.page_token;
530
531
  const seenTokens = new Set<string>();
531
532
  for (let page = 0; ; page++) {
@@ -549,6 +550,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
549
550
  // A provider page is atomic: `max_results` caps how much a run starts,
550
551
  // never how much of a fetched page is stored. Dropping a page's tail
551
552
  // would lose those events for good once the cursor moved past them.
553
+ const events: EventEnvelope[] = [];
552
554
  for (const calEvent of items) {
553
555
  const envelope = durableChanges
554
556
  ? this.calendarEventToChangeEnvelope(calEvent)
@@ -570,17 +572,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
570
572
  throw new Error('Google Calendar returned a repeated page token.');
571
573
  }
572
574
  pageToken = data.nextPageToken;
575
+ await ctx.commit(events, parked(pageToken));
576
+ collected += events.length;
573
577
 
574
578
  // Stop once this run has collected its share, once the provider returns
575
579
  // an empty page, or at the hard ceiling — the last of which is the only
576
580
  // bound on a window of nothing but filtered-out items, where
577
- // `events.length` never reaches the cap.
581
+ // `collected` never reaches the cap. The traversal continues next run.
578
582
  if (
579
- events.length >= maxResults ||
583
+ collected >= maxResults ||
580
584
  items.length === 0 ||
581
585
  page + 1 >= MAX_SYNC_PAGES
582
586
  ) {
583
- return park(pageToken);
587
+ return { status: 'more' };
584
588
  }
585
589
  }
586
590
  }
@@ -985,29 +989,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
985
989
  };
986
990
  }
987
991
 
988
- private buildResult(
989
- events: EventEnvelope[],
990
- syncToken: string | undefined
991
- ): SyncResult {
992
- // Sort events by occurred_at descending
993
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
994
-
992
+ private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
995
993
  // Built fresh rather than spread over the previous checkpoint: the whole
996
994
  // object replaces the stored one, so a run that recovered from a rejected
997
995
  // token cannot leave that token behind. When the sync produced no new token
998
996
  // the key is absent, and the next run correctly starts from a full sync.
999
- const newCheckpoint: CalendarCheckpoint = {
997
+ return {
1000
998
  ...(syncToken ? { sync_token: syncToken } : {}),
1001
999
  last_sync_at: new Date().toISOString(),
1002
1000
  };
1003
-
1004
- return {
1005
- events,
1006
- checkpoint: newCheckpoint as Record<string, unknown>,
1007
- metadata: {
1008
- items_found: events.length,
1009
- },
1010
- };
1011
1001
  }
1012
1002
 
1013
1003
  // Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
286
286
  // Connector
287
287
  // ---------------------------------------------------------------------------
288
288
 
289
+ type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
290
+
291
+ /**
292
+ * Checkpoint for a bootstrap with list pages still unread.
293
+ *
294
+ * `page_token` is deliberately absent: while it is missing the feed cannot go
295
+ * incremental, which is exactly the property that keeps the unread files
296
+ * reachable.
297
+ */
298
+ function bootstrapCheckpoint(
299
+ pendingPageToken: string,
300
+ listPageToken: string,
301
+ startedAt: string
302
+ ): DriveCheckpoint {
303
+ return {
304
+ pending_page_token: pendingPageToken,
305
+ list_page_token: listPageToken,
306
+ last_sync_at: startedAt,
307
+ };
308
+ }
309
+
289
310
  export default class GoogleDriveConnector extends ConnectorRuntime<
290
311
  Record<string, unknown>,
291
312
  DriveConfig
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
493
514
 
494
515
  // An unfinished bootstrap outranks everything: going incremental with pages
495
516
  // still unread would strand every file the traversal never reached.
517
+ const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
518
+ ctx.commit(events, next as Record<string, unknown>);
519
+
496
520
  if (checkpoint.list_page_token && checkpoint.pending_page_token) {
497
- return this.fullSync(http, config, {
521
+ return this.fullSync(http, config, commit, {
498
522
  startToken: checkpoint.pending_page_token,
499
523
  listPageToken: checkpoint.list_page_token,
500
524
  startedAt: checkpoint.last_sync_at,
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
506
530
  http,
507
531
  checkpoint.page_token,
508
532
  config,
533
+ commit,
509
534
  checkpoint.last_sync_at
510
535
  );
511
- if (result) {
512
- // A partial run must NOT advance `last_sync_at`. The view-churn guard
513
- // skips content for files modified before that stamp, so stamping now
514
- // would make the next run treat the unread backlog as already-stored
515
- // and drop its content on the floor.
516
- return this.buildResult(
517
- result.events,
518
- result.nextPageToken,
519
- result.partial ? checkpoint.last_sync_at : undefined,
520
- { changesPending: result.partial }
521
- );
522
- }
536
+ if (result) return result;
523
537
  // Token rejected (see isPageTokenRejection). Fall through to ONE full
524
538
  // re-list, exactly as the Calendar connector does — never a retry loop.
525
539
  }
526
540
 
527
- return this.fullSync(http, config);
541
+ return this.fullSync(http, config, commit);
528
542
  }
529
543
 
530
544
  /**
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
539
553
  private async fullSync(
540
554
  http: HttpClient,
541
555
  config: DriveConfig,
556
+ commit: DriveCommit,
542
557
  resume?: { startToken: string; listPageToken: string; startedAt?: string }
543
558
  ): Promise<SyncResult> {
544
559
  // A resumed bootstrap keeps the token minted when the traversal STARTED —
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
555
570
  const maxResults = Math.min(config.max_results ?? 500, 2000);
556
571
  const includeContent = config.include_content !== false;
557
572
  const q = buildListQuery(config);
558
- const events: EventEnvelope[] = [];
573
+ let collected = 0;
559
574
 
560
575
  // 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
561
576
  // configurable max_results. Defensive bound against a self-referential
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
605
620
  { maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
606
621
  );
607
622
 
623
+ // A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
624
+ // client-side against a change record, so the only way to keep honouring it
625
+ // is to keep listing. Withholding the token re-lists next run; promoting it
626
+ // would silently widen the feed to every changed file in the Drive.
627
+ const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
628
+
608
629
  const deadline = this.now() + SYNC_TIME_BUDGET_MS;
609
630
  for await (const items of pages) {
631
+ const events: EventEnvelope[] = [];
610
632
  for (const file of items) {
611
633
  events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
612
634
  }
613
- // Tested only BETWEEN pages: a page is consumed whole or not at all, so
614
- // `nextListPageToken` always addresses the first file we have not read.
615
- // Stopping on the clock parks a checkpoint; being killed on it does not.
616
- if (events.length >= maxResults || this.now() >= deadline) break;
617
- }
618
-
619
- // Unread pages mean the bootstrap is unfinished, and WHY it stopped is
620
- // irrelevant: the item cap, the clock, and the paginator's own MAX_PAGES
621
- // ceiling all leave files the change feed will never replay. Only Drive
622
- // withholding a nextPageToken proves the traversal exhausted, so that —
623
- // not the reason for stopping — is what may promote the change token.
624
- if (nextListPageToken) {
625
- return this.buildBootstrapResult(
626
- events,
627
- startToken,
628
- nextListPageToken,
629
- bootstrapStartedAt,
630
- searchIncomplete
631
- );
635
+ collected += events.length;
636
+ // Only Drive withholding a nextPageToken proves the traversal exhausted,
637
+ // so that — not the reason a run stops — is what may promote the change
638
+ // token. Until then each page commits with the bootstrap parked on the
639
+ // next unread page: a page is consumed whole or not at all, so the list
640
+ // token always addresses the first file not yet stored.
641
+ if (!nextListPageToken) {
642
+ await commit(events, {
643
+ ...(hasCustomQuery ? {} : { page_token: startToken }),
644
+ last_sync_at: bootstrapStartedAt,
645
+ });
646
+ return {
647
+ status: 'complete',
648
+ metadata: {
649
+ items_found: collected,
650
+ bootstrap_complete: true,
651
+ ...(searchIncomplete ? { search_incomplete: true } : {}),
652
+ },
653
+ };
654
+ }
655
+ await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
656
+ if (collected >= maxResults || this.now() >= deadline) break;
632
657
  }
633
658
 
634
- // A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
635
- // client-side against a change record, so the only way to keep honouring it
636
- // is to keep listing. Withholding the token re-lists next run; promoting it
637
- // would silently widen the feed to every changed file in the Drive.
638
- const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
639
- return this.buildResult(
640
- events,
641
- hasCustomQuery ? undefined : startToken,
642
- bootstrapStartedAt,
643
- { searchIncomplete }
644
- );
659
+ // Unread pages remain, and WHY this run stopped is irrelevant: the item
660
+ // cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
661
+ // the change feed will never replay. The bootstrap resumes next run.
662
+ return {
663
+ status: 'more',
664
+ metadata: {
665
+ items_found: collected,
666
+ bootstrap_complete: false,
667
+ ...(searchIncomplete ? { search_incomplete: true } : {}),
668
+ },
669
+ };
645
670
  }
646
671
 
647
672
  /**
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
654
679
  http: HttpClient,
655
680
  pageToken: string,
656
681
  config: DriveConfig,
682
+ commit: DriveCommit,
657
683
  lastSyncAt?: string
658
- ): Promise<{
659
- events: EventEnvelope[];
660
- nextPageToken?: string;
661
- partial: boolean;
662
- } | null> {
684
+ ): Promise<SyncResult | null> {
663
685
  const includeContent = config.include_content !== false;
664
686
  const includeTrashed = Boolean(config.include_trashed);
665
687
  const folderId =
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
667
689
  ? config.folder_id.trim()
668
690
  : undefined;
669
691
  const maxResults = Math.min(config.max_results ?? 500, 2000);
670
- const events: EventEnvelope[] = [];
692
+ let collected = 0;
671
693
  let nextPageToken: string | undefined;
672
694
  // The cursor for the page AFTER the one just consumed. A run that stops
673
695
  // early persists this instead of dropping back to a full re-list — a
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
718
740
  // start of the next page, so stopping part-way through one would skip the
719
741
  // changes still in it.
720
742
  const deadline = this.now() + SYNC_TIME_BUDGET_MS;
721
- let stoppedEarly = false;
722
743
  for await (const items of pages) {
744
+ if (rejected) return null;
745
+ const events: EventEnvelope[] = [];
723
746
  for (const change of items) {
724
747
  const envelope = await this.changeToEnvelope(
725
748
  http,
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
731
754
  );
732
755
  if (envelope) events.push(envelope);
733
756
  }
734
- if (rejected) break;
735
- if ((events.length >= maxResults || this.now() >= deadline) && resumeCursor) {
736
- stoppedEarly = true;
737
- break;
757
+ collected += events.length;
758
+ if (!resumeCursor) {
759
+ // The stream is caught up: the page that carried `newStartPageToken`
760
+ // commits it and stamps the run. Rebuilt rather than spread over the
761
+ // previous checkpoint so a run that recovered from a rejected token
762
+ // cannot leave that token behind; without a new token the key is
763
+ // absent and the next run correctly re-lists.
764
+ await commit(events, {
765
+ ...(nextPageToken ? { page_token: nextPageToken } : {}),
766
+ last_sync_at: new Date().toISOString(),
767
+ });
768
+ return {
769
+ status: 'complete',
770
+ metadata: { items_found: collected, bootstrap_complete: true },
771
+ };
738
772
  }
773
+ // Mid-stream: commit with the cursor of the next page, a resumable
774
+ // position in the change stream. `last_sync_at` does NOT advance — the
775
+ // view-churn guard skips content for files modified before that stamp,
776
+ // so stamping now would make the next run treat the unread backlog as
777
+ // already stored and drop its content on the floor.
778
+ await commit(events, {
779
+ page_token: resumeCursor,
780
+ ...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
781
+ });
782
+ if (collected >= maxResults || this.now() >= deadline) break;
739
783
  }
740
-
741
784
  if (rejected) return null;
742
785
 
743
- // Either we stopped on the cap, or the paginator ran out of pages with the
744
- // stream still open (MAX_PAGES). Both leave changes unread, and both must
745
- // hand back the cursor: discarding it would silently demote the next run to
746
- // a full re-list and restart the whole bootstrap.
747
- const unread = stoppedEarly || (!nextPageToken && Boolean(resumeCursor));
748
- if (unread && resumeCursor) {
749
- return { events, nextPageToken: resumeCursor, partial: true };
750
- }
751
-
752
- return { events, nextPageToken, partial: false };
786
+ // Stopped on the cap or the clock, or the paginator ran out of pages
787
+ // (MAX_PAGES) with the stream still open. The committed cursor resumes it.
788
+ return {
789
+ status: 'more',
790
+ metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
791
+ };
753
792
  }
754
793
 
755
794
  private async fetchStartPageToken(http: HttpClient): Promise<string> {
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
1219
1258
  };
1220
1259
  }
1221
1260
 
1222
- /**
1223
- * Checkpoint for a bootstrap that stopped on `max_results` with pages left.
1224
- *
1225
- * `page_token` is deliberately absent: while it is missing the feed cannot go
1226
- * incremental, which is exactly the property that keeps the unread files
1227
- * reachable. `bootstrap_complete: false` is surfaced in metadata so an
1228
- * operator can see the traversal is still catching up rather than guessing
1229
- * from a suspiciously round item count.
1230
- */
1231
- private buildBootstrapResult(
1232
- events: EventEnvelope[],
1233
- pendingPageToken: string,
1234
- listPageToken: string,
1235
- startedAt: string,
1236
- searchIncomplete = false
1237
- ): SyncResult {
1238
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
1239
-
1240
- const checkpoint: DriveCheckpoint = {
1241
- pending_page_token: pendingPageToken,
1242
- list_page_token: listPageToken,
1243
- last_sync_at: startedAt,
1244
- };
1245
-
1246
- return {
1247
- events,
1248
- checkpoint: checkpoint as Record<string, unknown>,
1249
- metadata: {
1250
- items_found: events.length,
1251
- bootstrap_complete: false,
1252
- ...(searchIncomplete ? { search_incomplete: true } : {}),
1253
- },
1254
- };
1255
- }
1256
-
1257
- private buildResult(
1258
- events: EventEnvelope[],
1259
- pageToken: string | undefined,
1260
- /** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
1261
- keepLastSyncAt: string | undefined,
1262
- flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
1263
- ): SyncResult {
1264
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
1265
-
1266
- // Rebuilt rather than spread over the previous checkpoint so a run that
1267
- // recovered from a rejected token cannot leave that token behind. Without a
1268
- // new token the key is absent and the next run correctly re-lists.
1269
- const newCheckpoint: DriveCheckpoint = {
1270
- ...(pageToken ? { page_token: pageToken } : {}),
1271
- last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
1272
- };
1273
-
1274
- return {
1275
- events,
1276
- checkpoint: newCheckpoint as Record<string, unknown>,
1277
- metadata: {
1278
- items_found: events.length,
1279
- bootstrap_complete: true,
1280
- ...(flags.changesPending ? { changes_pending: true } : {}),
1281
- ...(flags.searchIncomplete ? { search_incomplete: true } : {}),
1282
- },
1283
- };
1284
- }
1285
-
1286
1261
  /** Overridable so a test can drive the time budget without sleeping. */
1287
1262
  protected now(): number {
1288
1263
  return Date.now();