@lobu/runtime-device 20.2.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +42 -36
  3. package/npm-shrinkwrap.json +270 -224
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/config/define.d.ts +25 -8
  31. package/vendor/cli/dist/config/define.js +24 -0
  32. package/vendor/cli/dist/connectors/README.md +15 -16
  33. package/vendor/cli/dist/connectors/github.ts +15 -9
  34. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  35. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  36. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  37. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  38. package/vendor/cli/dist/connectors/jira.ts +62 -27
  39. package/vendor/cli/dist/connectors/linear.ts +55 -21
  40. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  41. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  42. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  43. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  44. package/vendor/cli/dist/connectors/rss.ts +2 -2
  45. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  46. package/vendor/cli/dist/connectors/x.ts +32 -17
  47. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  48. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  49. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  50. package/vendor/cli/dist/runtime-components.json +4 -4
  51. package/vendor/cli/package.json +2 -1
  52. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  53. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  54. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  55. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  56. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  57. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  58. package/vendor/connector-sdk/dist/http-client.js +15 -13
  59. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  60. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  61. package/vendor/connector-sdk/dist/retry.js +40 -64
  62. package/vendor/connector-sdk/package.json +1 -1
  63. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  64. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  65. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  66. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  67. package/vendor/connector-worker/dist/compile/index.js +41 -8
  68. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  69. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  70. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  71. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  72. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  73. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  74. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  75. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  76. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  77. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  78. package/vendor/connector-worker/package.json +1 -1
  79. package/vendor/core/dist/agent-policy.js +3 -0
  80. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  81. package/vendor/core/dist/connector-query-errors.js +8 -3
  82. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  83. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  84. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  85. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  86. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  87. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  88. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  89. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  90. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  91. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  92. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  93. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  94. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  95. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  96. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  97. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  98. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  99. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  100. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  101. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  102. package/vendor/core/dist/reserved.d.ts +3 -3
  103. package/vendor/core/dist/reserved.js +9 -1
  104. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  105. package/vendor/core/package.json +25 -5
  106. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  107. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -457,7 +457,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
457
457
  const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
458
458
 
459
459
  const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
460
- const events: EventEnvelope[] = [];
461
460
  const durableChanges = ctx.feedKey === 'changes';
462
461
 
463
462
  // A checkpoint minted under a different scope (or by an older version that
@@ -467,10 +466,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
467
466
  const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
468
467
  const resumable = checkpoint.scope === scope;
469
468
 
470
- /** Traversal reached the last page: store its cursor and stamp the run. */
471
- const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
472
- const result = this.buildResult(items, syncToken);
473
- return { ...result, checkpoint: { ...result.checkpoint, scope } };
469
+ /** Traversal reached the last page: commit it with its cursor and stamp the run. */
470
+ const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
471
+ await ctx.commit(items, {
472
+ ...this.completedCheckpoint(syncToken),
473
+ scope,
474
+ } as Record<string, unknown>);
475
+ return { status: 'complete' };
474
476
  };
475
477
 
476
478
  if (resumable && checkpoint.sync_token) {
@@ -517,15 +519,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
517
519
  const baseParams = params.toString();
518
520
 
519
521
  /**
520
- * Ran out of budget mid-traversal: park the next page token and return no
521
- * sync token and no `last_sync_at`, so nothing downstream can read this run
522
- * as a completed window. The next scheduled run resumes from `token`.
522
+ * Mid-traversal: the next page token, with no sync token and no
523
+ * `last_sync_at`, so nothing downstream can read it as a completed window.
524
+ * A run that stops here resumes from `token`.
523
525
  */
524
- const park = (token: string): SyncResult => ({
525
- ...this.buildResult(events, undefined),
526
- checkpoint: { scope, pending: { params: baseParams, page_token: token } },
527
- });
526
+ const parked = (token: string) =>
527
+ ({ scope, pending: { params: baseParams, page_token: token } }) as Record<string, unknown>;
528
528
 
529
+ let collected = 0;
529
530
  let pageToken = pending?.page_token;
530
531
  const seenTokens = new Set<string>();
531
532
  for (let page = 0; ; page++) {
@@ -549,6 +550,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
549
550
  // A provider page is atomic: `max_results` caps how much a run starts,
550
551
  // never how much of a fetched page is stored. Dropping a page's tail
551
552
  // would lose those events for good once the cursor moved past them.
553
+ const events: EventEnvelope[] = [];
552
554
  for (const calEvent of items) {
553
555
  const envelope = durableChanges
554
556
  ? this.calendarEventToChangeEnvelope(calEvent)
@@ -570,17 +572,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
570
572
  throw new Error('Google Calendar returned a repeated page token.');
571
573
  }
572
574
  pageToken = data.nextPageToken;
575
+ await ctx.commit(events, parked(pageToken));
576
+ collected += events.length;
573
577
 
574
578
  // Stop once this run has collected its share, once the provider returns
575
579
  // an empty page, or at the hard ceiling — the last of which is the only
576
580
  // bound on a window of nothing but filtered-out items, where
577
- // `events.length` never reaches the cap.
581
+ // `collected` never reaches the cap. The traversal continues next run.
578
582
  if (
579
- events.length >= maxResults ||
583
+ collected >= maxResults ||
580
584
  items.length === 0 ||
581
585
  page + 1 >= MAX_SYNC_PAGES
582
586
  ) {
583
- return park(pageToken);
587
+ return { status: 'more' };
584
588
  }
585
589
  }
586
590
  }
@@ -985,29 +989,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
985
989
  };
986
990
  }
987
991
 
988
- private buildResult(
989
- events: EventEnvelope[],
990
- syncToken: string | undefined
991
- ): SyncResult {
992
- // Sort events by occurred_at descending
993
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
994
-
992
+ private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
995
993
  // Built fresh rather than spread over the previous checkpoint: the whole
996
994
  // object replaces the stored one, so a run that recovered from a rejected
997
995
  // token cannot leave that token behind. When the sync produced no new token
998
996
  // the key is absent, and the next run correctly starts from a full sync.
999
- const newCheckpoint: CalendarCheckpoint = {
997
+ return {
1000
998
  ...(syncToken ? { sync_token: syncToken } : {}),
1001
999
  last_sync_at: new Date().toISOString(),
1002
1000
  };
1003
-
1004
- return {
1005
- events,
1006
- checkpoint: newCheckpoint as Record<string, unknown>,
1007
- metadata: {
1008
- items_found: events.length,
1009
- },
1010
- };
1011
1001
  }
1012
1002
 
1013
1003
  // Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
286
286
  // Connector
287
287
  // ---------------------------------------------------------------------------
288
288
 
289
+ type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
290
+
291
+ /**
292
+ * Checkpoint for a bootstrap with list pages still unread.
293
+ *
294
+ * `page_token` is deliberately absent: while it is missing the feed cannot go
295
+ * incremental, which is exactly the property that keeps the unread files
296
+ * reachable.
297
+ */
298
+ function bootstrapCheckpoint(
299
+ pendingPageToken: string,
300
+ listPageToken: string,
301
+ startedAt: string
302
+ ): DriveCheckpoint {
303
+ return {
304
+ pending_page_token: pendingPageToken,
305
+ list_page_token: listPageToken,
306
+ last_sync_at: startedAt,
307
+ };
308
+ }
309
+
289
310
  export default class GoogleDriveConnector extends ConnectorRuntime<
290
311
  Record<string, unknown>,
291
312
  DriveConfig
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
493
514
 
494
515
  // An unfinished bootstrap outranks everything: going incremental with pages
495
516
  // still unread would strand every file the traversal never reached.
517
+ const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
518
+ ctx.commit(events, next as Record<string, unknown>);
519
+
496
520
  if (checkpoint.list_page_token && checkpoint.pending_page_token) {
497
- return this.fullSync(http, config, {
521
+ return this.fullSync(http, config, commit, {
498
522
  startToken: checkpoint.pending_page_token,
499
523
  listPageToken: checkpoint.list_page_token,
500
524
  startedAt: checkpoint.last_sync_at,
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
506
530
  http,
507
531
  checkpoint.page_token,
508
532
  config,
533
+ commit,
509
534
  checkpoint.last_sync_at
510
535
  );
511
- if (result) {
512
- // A partial run must NOT advance `last_sync_at`. The view-churn guard
513
- // skips content for files modified before that stamp, so stamping now
514
- // would make the next run treat the unread backlog as already-stored
515
- // and drop its content on the floor.
516
- return this.buildResult(
517
- result.events,
518
- result.nextPageToken,
519
- result.partial ? checkpoint.last_sync_at : undefined,
520
- { changesPending: result.partial }
521
- );
522
- }
536
+ if (result) return result;
523
537
  // Token rejected (see isPageTokenRejection). Fall through to ONE full
524
538
  // re-list, exactly as the Calendar connector does — never a retry loop.
525
539
  }
526
540
 
527
- return this.fullSync(http, config);
541
+ return this.fullSync(http, config, commit);
528
542
  }
529
543
 
530
544
  /**
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
539
553
  private async fullSync(
540
554
  http: HttpClient,
541
555
  config: DriveConfig,
556
+ commit: DriveCommit,
542
557
  resume?: { startToken: string; listPageToken: string; startedAt?: string }
543
558
  ): Promise<SyncResult> {
544
559
  // A resumed bootstrap keeps the token minted when the traversal STARTED —
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
555
570
  const maxResults = Math.min(config.max_results ?? 500, 2000);
556
571
  const includeContent = config.include_content !== false;
557
572
  const q = buildListQuery(config);
558
- const events: EventEnvelope[] = [];
573
+ let collected = 0;
559
574
 
560
575
  // 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
561
576
  // configurable max_results. Defensive bound against a self-referential
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
605
620
  { maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
606
621
  );
607
622
 
623
+ // A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
624
+ // client-side against a change record, so the only way to keep honouring it
625
+ // is to keep listing. Withholding the token re-lists next run; promoting it
626
+ // would silently widen the feed to every changed file in the Drive.
627
+ const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
628
+
608
629
  const deadline = this.now() + SYNC_TIME_BUDGET_MS;
609
630
  for await (const items of pages) {
631
+ const events: EventEnvelope[] = [];
610
632
  for (const file of items) {
611
633
  events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
612
634
  }
613
- // Tested only BETWEEN pages: a page is consumed whole or not at all, so
614
- // `nextListPageToken` always addresses the first file we have not read.
615
- // Stopping on the clock parks a checkpoint; being killed on it does not.
616
- if (events.length >= maxResults || this.now() >= deadline) break;
617
- }
618
-
619
- // Unread pages mean the bootstrap is unfinished, and WHY it stopped is
620
- // irrelevant: the item cap, the clock, and the paginator's own MAX_PAGES
621
- // ceiling all leave files the change feed will never replay. Only Drive
622
- // withholding a nextPageToken proves the traversal exhausted, so that —
623
- // not the reason for stopping — is what may promote the change token.
624
- if (nextListPageToken) {
625
- return this.buildBootstrapResult(
626
- events,
627
- startToken,
628
- nextListPageToken,
629
- bootstrapStartedAt,
630
- searchIncomplete
631
- );
635
+ collected += events.length;
636
+ // Only Drive withholding a nextPageToken proves the traversal exhausted,
637
+ // so that — not the reason a run stops — is what may promote the change
638
+ // token. Until then each page commits with the bootstrap parked on the
639
+ // next unread page: a page is consumed whole or not at all, so the list
640
+ // token always addresses the first file not yet stored.
641
+ if (!nextListPageToken) {
642
+ await commit(events, {
643
+ ...(hasCustomQuery ? {} : { page_token: startToken }),
644
+ last_sync_at: bootstrapStartedAt,
645
+ });
646
+ return {
647
+ status: 'complete',
648
+ metadata: {
649
+ items_found: collected,
650
+ bootstrap_complete: true,
651
+ ...(searchIncomplete ? { search_incomplete: true } : {}),
652
+ },
653
+ };
654
+ }
655
+ await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
656
+ if (collected >= maxResults || this.now() >= deadline) break;
632
657
  }
633
658
 
634
- // A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
635
- // client-side against a change record, so the only way to keep honouring it
636
- // is to keep listing. Withholding the token re-lists next run; promoting it
637
- // would silently widen the feed to every changed file in the Drive.
638
- const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
639
- return this.buildResult(
640
- events,
641
- hasCustomQuery ? undefined : startToken,
642
- bootstrapStartedAt,
643
- { searchIncomplete }
644
- );
659
+ // Unread pages remain, and WHY this run stopped is irrelevant: the item
660
+ // cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
661
+ // the change feed will never replay. The bootstrap resumes next run.
662
+ return {
663
+ status: 'more',
664
+ metadata: {
665
+ items_found: collected,
666
+ bootstrap_complete: false,
667
+ ...(searchIncomplete ? { search_incomplete: true } : {}),
668
+ },
669
+ };
645
670
  }
646
671
 
647
672
  /**
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
654
679
  http: HttpClient,
655
680
  pageToken: string,
656
681
  config: DriveConfig,
682
+ commit: DriveCommit,
657
683
  lastSyncAt?: string
658
- ): Promise<{
659
- events: EventEnvelope[];
660
- nextPageToken?: string;
661
- partial: boolean;
662
- } | null> {
684
+ ): Promise<SyncResult | null> {
663
685
  const includeContent = config.include_content !== false;
664
686
  const includeTrashed = Boolean(config.include_trashed);
665
687
  const folderId =
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
667
689
  ? config.folder_id.trim()
668
690
  : undefined;
669
691
  const maxResults = Math.min(config.max_results ?? 500, 2000);
670
- const events: EventEnvelope[] = [];
692
+ let collected = 0;
671
693
  let nextPageToken: string | undefined;
672
694
  // The cursor for the page AFTER the one just consumed. A run that stops
673
695
  // early persists this instead of dropping back to a full re-list — a
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
718
740
  // start of the next page, so stopping part-way through one would skip the
719
741
  // changes still in it.
720
742
  const deadline = this.now() + SYNC_TIME_BUDGET_MS;
721
- let stoppedEarly = false;
722
743
  for await (const items of pages) {
744
+ if (rejected) return null;
745
+ const events: EventEnvelope[] = [];
723
746
  for (const change of items) {
724
747
  const envelope = await this.changeToEnvelope(
725
748
  http,
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
731
754
  );
732
755
  if (envelope) events.push(envelope);
733
756
  }
734
- if (rejected) break;
735
- if ((events.length >= maxResults || this.now() >= deadline) && resumeCursor) {
736
- stoppedEarly = true;
737
- break;
757
+ collected += events.length;
758
+ if (!resumeCursor) {
759
+ // The stream is caught up: the page that carried `newStartPageToken`
760
+ // commits it and stamps the run. Rebuilt rather than spread over the
761
+ // previous checkpoint so a run that recovered from a rejected token
762
+ // cannot leave that token behind; without a new token the key is
763
+ // absent and the next run correctly re-lists.
764
+ await commit(events, {
765
+ ...(nextPageToken ? { page_token: nextPageToken } : {}),
766
+ last_sync_at: new Date().toISOString(),
767
+ });
768
+ return {
769
+ status: 'complete',
770
+ metadata: { items_found: collected, bootstrap_complete: true },
771
+ };
738
772
  }
773
+ // Mid-stream: commit with the cursor of the next page, a resumable
774
+ // position in the change stream. `last_sync_at` does NOT advance — the
775
+ // view-churn guard skips content for files modified before that stamp,
776
+ // so stamping now would make the next run treat the unread backlog as
777
+ // already stored and drop its content on the floor.
778
+ await commit(events, {
779
+ page_token: resumeCursor,
780
+ ...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
781
+ });
782
+ if (collected >= maxResults || this.now() >= deadline) break;
739
783
  }
740
-
741
784
  if (rejected) return null;
742
785
 
743
- // Either we stopped on the cap, or the paginator ran out of pages with the
744
- // stream still open (MAX_PAGES). Both leave changes unread, and both must
745
- // hand back the cursor: discarding it would silently demote the next run to
746
- // a full re-list and restart the whole bootstrap.
747
- const unread = stoppedEarly || (!nextPageToken && Boolean(resumeCursor));
748
- if (unread && resumeCursor) {
749
- return { events, nextPageToken: resumeCursor, partial: true };
750
- }
751
-
752
- return { events, nextPageToken, partial: false };
786
+ // Stopped on the cap or the clock, or the paginator ran out of pages
787
+ // (MAX_PAGES) with the stream still open. The committed cursor resumes it.
788
+ return {
789
+ status: 'more',
790
+ metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
791
+ };
753
792
  }
754
793
 
755
794
  private async fetchStartPageToken(http: HttpClient): Promise<string> {
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
1219
1258
  };
1220
1259
  }
1221
1260
 
1222
- /**
1223
- * Checkpoint for a bootstrap that stopped on `max_results` with pages left.
1224
- *
1225
- * `page_token` is deliberately absent: while it is missing the feed cannot go
1226
- * incremental, which is exactly the property that keeps the unread files
1227
- * reachable. `bootstrap_complete: false` is surfaced in metadata so an
1228
- * operator can see the traversal is still catching up rather than guessing
1229
- * from a suspiciously round item count.
1230
- */
1231
- private buildBootstrapResult(
1232
- events: EventEnvelope[],
1233
- pendingPageToken: string,
1234
- listPageToken: string,
1235
- startedAt: string,
1236
- searchIncomplete = false
1237
- ): SyncResult {
1238
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
1239
-
1240
- const checkpoint: DriveCheckpoint = {
1241
- pending_page_token: pendingPageToken,
1242
- list_page_token: listPageToken,
1243
- last_sync_at: startedAt,
1244
- };
1245
-
1246
- return {
1247
- events,
1248
- checkpoint: checkpoint as Record<string, unknown>,
1249
- metadata: {
1250
- items_found: events.length,
1251
- bootstrap_complete: false,
1252
- ...(searchIncomplete ? { search_incomplete: true } : {}),
1253
- },
1254
- };
1255
- }
1256
-
1257
- private buildResult(
1258
- events: EventEnvelope[],
1259
- pageToken: string | undefined,
1260
- /** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
1261
- keepLastSyncAt: string | undefined,
1262
- flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
1263
- ): SyncResult {
1264
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
1265
-
1266
- // Rebuilt rather than spread over the previous checkpoint so a run that
1267
- // recovered from a rejected token cannot leave that token behind. Without a
1268
- // new token the key is absent and the next run correctly re-lists.
1269
- const newCheckpoint: DriveCheckpoint = {
1270
- ...(pageToken ? { page_token: pageToken } : {}),
1271
- last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
1272
- };
1273
-
1274
- return {
1275
- events,
1276
- checkpoint: newCheckpoint as Record<string, unknown>,
1277
- metadata: {
1278
- items_found: events.length,
1279
- bootstrap_complete: true,
1280
- ...(flags.changesPending ? { changes_pending: true } : {}),
1281
- ...(flags.searchIncomplete ? { search_incomplete: true } : {}),
1282
- },
1283
- };
1284
- }
1285
-
1286
1261
  /** Overridable so a test can drive the time budget without sleeping. */
1287
1262
  protected now(): number {
1288
1263
  return Date.now();
@@ -100,6 +100,8 @@ interface GmailCheckpoint {
100
100
  /** Serialized search + person filter the two fields below were produced under. */
101
101
  scope?: string;
102
102
  last_sync_at?: string;
103
+ /** Undecodable threads, keyed by thread ID; retained until a later successful read. */
104
+ skipped_threads?: Record<string, { message_id: string; error: string }>;
103
105
  /** Set while a window is only part-walked: the run hit `max_results`, or a page came back empty with a cursor. */
104
106
  pending?: {
105
107
  query: string;
@@ -108,6 +110,12 @@ interface GmailCheckpoint {
108
110
  };
109
111
  }
110
112
 
113
+ class GmailBodyDecodeError extends Error {
114
+ constructor(readonly messageId: string, encoding: string, cause: unknown) {
115
+ super(`Gmail body decode (${encoding}): ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
116
+ }
117
+ }
118
+
111
119
  interface GmailConfig {
112
120
  /**
113
121
  * Gmail search scope shared by sync and direct source reads. Takes precedence
@@ -449,7 +457,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
449
457
  // sync
450
458
  // -------------------------------------------------------------------------
451
459
 
452
- private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult<GmailCheckpoint>> {
460
+ private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult> {
453
461
  const syncStartedAt = new Date();
454
462
  const token = ctx.credentials?.accessToken;
455
463
  if (!token) {
@@ -485,7 +493,23 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
485
493
  // first page. `origin_id` makes that replay a supersede, not a duplicate.
486
494
  let staleCursorRecoverable = pageToken !== undefined;
487
495
  const http = this.createClient(token);
488
- const events: EventEnvelope[] = [];
496
+ const skippedThreads = { ...checkpoint.skipped_threads };
497
+ let itemsFound = 0;
498
+ let itemsSkipped = 0;
499
+ // A part-walked window keeps the previous `last_sync_at`: it may not advance
500
+ // until the whole window has been walked, or the unread tail is skipped.
501
+ const checkpointAt = (cursor: string | undefined): GmailCheckpoint => {
502
+ const next: GmailCheckpoint = cursor
503
+ ? {
504
+ schema_version: 3,
505
+ scope: scopeKey,
506
+ ...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
507
+ pending: { query, started_at: windowStart, page_token: cursor },
508
+ }
509
+ : { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
510
+ if (Object.keys(skippedThreads).length) next.skipped_threads = { ...skippedThreads };
511
+ return next;
512
+ };
489
513
  // Bounds the threads FETCHED per run (each costs at least one API call),
490
514
  // independent of how many survive the person filter — a narrow feed must not
491
515
  // scan the whole window just because most threads are rejected. It also sizes
@@ -513,6 +537,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
513
537
  staleCursorRecoverable = false;
514
538
  const listData = (await listResponse.json()) as GmailThreadListResponse;
515
539
  const threads = listData.threads ?? [];
540
+ const events: EventEnvelope[] = [];
516
541
  for (const threadStub of threads) {
517
542
  inspected++;
518
543
  try {
@@ -562,6 +587,13 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
562
587
  ...(attribution.fromName ? { from_name: attribution.fromName } : {}),
563
588
  },
564
589
  });
590
+ delete skippedThreads[threadStub.id];
591
+ } catch (error) {
592
+ // Only decoding failures are permanent for these bytes. Fetch and
593
+ // conversion failures must still retry the window without advancing.
594
+ if (!(error instanceof GmailBodyDecodeError)) throw error;
595
+ skippedThreads[threadStub.id] = { message_id: error.messageId, error: error.message };
596
+ itemsSkipped++;
565
597
  } finally {
566
598
  await sleep(this.RATE_LIMIT_MS);
567
599
  }
@@ -570,31 +602,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
570
602
  throw new Error('Gmail returned a repeated page token');
571
603
  }
572
604
  pageToken = listData.nextPageToken;
573
- // An empty page with a cursor is still incomplete. Persist its cursor
574
- // rather than advance the window or spin on empty provider responses.
575
- if (threads.length === 0) break;
576
- if (!pageToken || inspected >= maxResults) break;
605
+ // The page commits with the cursor of the next one, so a run cut short
606
+ // resumes where it stopped; the last page commits the finished window.
607
+ await ctx.commit(events, checkpointAt(pageToken));
608
+ itemsFound += events.length;
609
+ const metadata = { items_found: itemsFound, items_skipped: itemsSkipped };
610
+ if (!pageToken) return { status: 'complete', metadata };
611
+ // An empty page with a cursor is still incomplete. Its cursor is saved
612
+ // and the next run continues immediately rather than reporting the
613
+ // fixed window as caught up.
614
+ if (threads.length === 0) return { status: 'more', metadata };
615
+ if (inspected >= maxResults) return { status: 'more', metadata };
577
616
  }
578
-
579
- events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
580
- // A part-walked window keeps the previous `last_sync_at`: it may not advance
581
- // until the whole window has been walked, or the unread tail is skipped.
582
- const newCheckpoint: GmailCheckpoint = pageToken
583
- ? {
584
- schema_version: 3,
585
- scope: scopeKey,
586
- ...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
587
- pending: { query, started_at: windowStart, page_token: pageToken },
588
- }
589
- : { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
590
-
591
- return {
592
- events,
593
- checkpoint: newCheckpoint,
594
- metadata: {
595
- items_found: events.length,
596
- },
597
- };
598
617
  }
599
618
 
600
619
  // -------------------------------------------------------------------------
@@ -1356,7 +1375,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
1356
1375
  if (mimeType !== 'text/plain' && mimeType !== 'text/html') return '';
1357
1376
  let text = '';
1358
1377
  if (payload.body?.data) {
1359
- text = this.base64UrlDecode(payload.body.data, payload);
1378
+ text = this.base64UrlDecode(payload.body.data, payload, messageId);
1360
1379
  } else if (payload.body?.attachmentId) {
1361
1380
  // Gmail may externalize the body itself, even under format=full. A
1362
1381
  // failed fetch must not silently replace that body with its alternative.
@@ -1364,7 +1383,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
1364
1383
  if (!response.ok) throw new Error(`Gmail message body error (${response.status}): ${await response.text()}`);
1365
1384
  const body = (await response.json()) as { data?: string };
1366
1385
  if (typeof body.data !== 'string') throw new Error('Gmail returned a message body without data');
1367
- text = this.base64UrlDecode(body.data, payload);
1386
+ text = this.base64UrlDecode(body.data, payload, messageId);
1368
1387
  }
1369
1388
  if (!text && payload.body?.size) throw new Error('Gmail returned a nonempty message body without data');
1370
1389
  // Convert each HTML leaf after charset decoding, preserving plain parts,
@@ -1374,14 +1393,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
1374
1393
  return text.trim() ? text : '';
1375
1394
  }
1376
1395
 
1377
- private base64UrlDecode(data: string, payload: GmailMessagePayload): string {
1396
+ private base64UrlDecode(data: string, payload: GmailMessagePayload, messageId: string): string {
1378
1397
  const padded = data.replace(/-/g, '+').replace(/_/g, '/');
1379
1398
  const contentType = payload.headers?.find((header) => header.name.toLowerCase() === 'content-type')?.value;
1380
1399
  const charset = contentType?.match(/;\s*charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\s]+))/i);
1381
1400
  const encoding = (charset?.[1] ?? charset?.[2] ?? charset?.[3] ?? 'utf-8').trim();
1382
1401
  // MIME bodies contain bytes in the part's declared charset. Invalid bytes
1383
1402
  // or unsupported labels must fail the read instead of persisting corruption.
1384
- return new TextDecoder(encoding, { fatal: true }).decode(Buffer.from(padded, 'base64'));
1403
+ try {
1404
+ return new TextDecoder(encoding, { fatal: true }).decode(Buffer.from(padded, 'base64'));
1405
+ } catch (cause) {
1406
+ throw new GmailBodyDecodeError(messageId, encoding, cause);
1407
+ }
1385
1408
  }
1386
1409
 
1387
1410
  private base64UrlEncode(str: string): string {
@@ -376,11 +376,8 @@ export default class HackerNewsConnector extends ConnectorRuntime {
376
376
  await this.enrichStoriesWithExternalContent(events, deadline);
377
377
  }
378
378
 
379
- return {
380
- events,
381
- checkpoint: { last_sync_at: new Date().toISOString() },
382
- metadata: { items_found: events.length },
383
- };
379
+ await ctx.commit(events, { last_sync_at: new Date().toISOString() });
380
+ return { status: 'complete', metadata: { items_found: events.length } };
384
381
  }
385
382
 
386
383
  // -------------------------------------------------------------------------