@lobu/runtime-device 20.1.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +46 -40
  3. package/npm-shrinkwrap.json +283 -237
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/commands/login.js +3 -0
  31. package/vendor/cli/dist/config/define.d.ts +25 -8
  32. package/vendor/cli/dist/config/define.js +24 -0
  33. package/vendor/cli/dist/connectors/README.md +15 -16
  34. package/vendor/cli/dist/connectors/chat-user-identity.ts +41 -0
  35. package/vendor/cli/dist/connectors/github.ts +15 -9
  36. package/vendor/cli/dist/connectors/google-identity.ts +11 -0
  37. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  38. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  39. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  40. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  41. package/vendor/cli/dist/connectors/jira.ts +62 -27
  42. package/vendor/cli/dist/connectors/linear.ts +55 -21
  43. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  44. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  45. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  46. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  47. package/vendor/cli/dist/connectors/rss.ts +2 -2
  48. package/vendor/cli/dist/connectors/slack-identity.ts +19 -0
  49. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  50. package/vendor/cli/dist/connectors/x.ts +32 -17
  51. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  52. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  53. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  54. package/vendor/cli/dist/internal/credentials.js +5 -1
  55. package/vendor/cli/dist/internal/oauth.d.ts +15 -1
  56. package/vendor/cli/dist/internal/oauth.js +9 -1
  57. package/vendor/cli/dist/runtime-components.json +4 -4
  58. package/vendor/cli/package.json +2 -1
  59. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  60. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  61. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  62. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  63. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  64. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  65. package/vendor/connector-sdk/dist/http-client.js +15 -13
  66. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  67. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  68. package/vendor/connector-sdk/dist/retry.js +40 -64
  69. package/vendor/connector-sdk/package.json +1 -1
  70. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  71. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  72. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  73. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  74. package/vendor/connector-worker/dist/compile/index.js +41 -8
  75. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  76. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  77. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  78. package/vendor/connector-worker/dist/daemon/automation-process.d.ts +32 -0
  79. package/vendor/connector-worker/dist/daemon/automation-process.js +102 -1
  80. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  81. package/vendor/connector-worker/dist/daemon/builtins/index.js +8 -0
  82. package/vendor/connector-worker/dist/daemon/builtins/os-shell.d.ts +13 -1
  83. package/vendor/connector-worker/dist/daemon/builtins/os-shell.js +28 -2
  84. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  85. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  86. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  87. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  88. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  89. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  90. package/vendor/connector-worker/package.json +1 -1
  91. package/vendor/core/dist/agent-policy.js +3 -0
  92. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  93. package/vendor/core/dist/connector-query-errors.js +8 -3
  94. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  95. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  96. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  97. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  98. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  99. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  100. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  101. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  102. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  103. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  104. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  105. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  106. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  107. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  108. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  109. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  110. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  111. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  112. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  113. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  114. package/vendor/core/dist/credentials.d.ts +16 -1
  115. package/vendor/core/dist/credentials.js +10 -1
  116. package/vendor/core/dist/reserved.d.ts +3 -3
  117. package/vendor/core/dist/reserved.js +9 -1
  118. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  119. package/vendor/core/package.json +25 -5
  120. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  121. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -14,9 +14,9 @@ import {
14
14
  type EventEnvelope,
15
15
  fileDownloadOutput,
16
16
  type HttpClient,
17
+ HttpStatusError,
17
18
  inlineContentBudget,
18
19
  inlineMaxBytesSchema,
19
- paginateByCursor,
20
20
  requireBearerClient,
21
21
  type SyncContext,
22
22
  type SyncResult,
@@ -89,7 +89,24 @@ interface GraphPagedResponse<T> {
89
89
  // ---------------------------------------------------------------------------
90
90
 
91
91
  interface OutlookCheckpoint {
92
+ /** When the last complete traversal began; the next window starts here. */
92
93
  last_sync_at?: string;
94
+ /**
95
+ * A traversal that stopped at the per-run cap with pages left. `window`
96
+ * is the fixed range it walks and `next_link` Graph's cursor for the first
97
+ * page not yet stored, so the next run continues exactly where this one
98
+ * stopped instead of re-reading the newest page and never reaching the rest.
99
+ */
100
+ pending?: { window: { start: string; end: string }; next_link: string };
101
+ }
102
+
103
+ const WINDOW_OVERLAP_MS = 5 * 60_000;
104
+
105
+ /** ISO timestamp `days` from now (negative for the past). */
106
+ function daysFromNow(days: number): string {
107
+ const date = new Date();
108
+ date.setDate(date.getDate() + days);
109
+ return date.toISOString();
93
110
  }
94
111
 
95
112
  // ---------------------------------------------------------------------------
@@ -448,67 +465,53 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
448
465
  private async syncMessages(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
449
466
  const config = ctx.config as Record<string, unknown>;
450
467
  const folder = (config.folder as string) ?? 'inbox';
451
- const maxResults = (config.max_results as number) ?? 50;
452
468
  const lookbackDays = (config.lookback_days as number) ?? 30;
453
- const encodedFolder = encodeURIComponent(folder);
454
-
455
- const since = new Date();
456
- since.setDate(since.getDate() - lookbackDays);
457
- const sinceFilter = since.toISOString();
458
-
459
- const events: EventEnvelope[] = [];
469
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
470
+
471
+ // A fixed window per traversal: from where the last complete one began
472
+ // (or the lookback) up to when this one began. Mail arriving meanwhile
473
+ // belongs to the next window, so a capped walk can never be pushed past
474
+ // messages it has not reached by newer arrivals landing at the front.
475
+ // The start overlaps the previous window by a few minutes: Graph can
476
+ // surface a message after its receivedDateTime, and `origin_id` makes the
477
+ // replayed boundary a supersede, not a duplicate.
478
+ const window = checkpoint.pending?.window ?? {
479
+ start: checkpoint.last_sync_at
480
+ ? new Date(Date.parse(checkpoint.last_sync_at) - WINDOW_OVERLAP_MS).toISOString()
481
+ : daysFromNow(-lookbackDays),
482
+ end: new Date().toISOString(),
483
+ };
460
484
  const firstUrl =
461
- `${this.API_BASE}/me/mailFolders/${encodedFolder}/messages` +
462
- `?$top=${Math.min(maxResults, this.PAGE_SIZE)}` +
485
+ `${this.API_BASE}/me/mailFolders/${encodeURIComponent(folder)}/messages` +
486
+ `?$top=${this.PAGE_SIZE}` +
463
487
  '&$orderby=receivedDateTime desc' +
464
- `&$filter=receivedDateTime ge ${sinceFilter}` +
488
+ `&$filter=receivedDateTime ge ${window.start} and receivedDateTime lt ${window.end}` +
465
489
  '&$select=id,conversationId,subject,bodyPreview,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,importance,isRead,webLink';
466
490
 
467
- let fetched = 0;
468
-
469
- // Graph paginates via full `@odata.nextLink` URLs, so the cursor is the page URL.
470
- const pages = paginateByCursor<GraphMessage, string>(
471
- async (url) => {
472
- const data = await http.get<GraphPagedResponse<GraphMessage>>(url ?? firstUrl);
473
- return { items: data.value, nextCursor: data['@odata.nextLink'] };
474
- },
475
- { maxPages: this.MAX_PAGES, initialCursor: firstUrl }
476
- );
477
-
478
- for await (const value of pages) {
479
- for (const msg of value) {
480
- if (fetched >= maxResults) break;
481
- events.push({
482
- origin_id: `outlook_msg_${msg.id}`,
483
- title: msg.subject,
484
- payload_text: msg.bodyPreview || msg.subject,
485
- author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
486
- source_url: msg.webLink,
487
- occurred_at: new Date(msg.receivedDateTime),
488
- origin_type: 'email',
489
- metadata: {
490
- from: msg.from?.emailAddress?.address,
491
- to: formatRecipients(msg.toRecipients ?? []),
492
- cc: formatRecipients(msg.ccRecipients ?? []),
493
- importance: msg.importance,
494
- has_attachments: msg.hasAttachments,
495
- is_read: msg.isRead,
496
- },
497
- });
498
- fetched++;
499
- }
500
-
501
- if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
502
-
503
- if (fetched >= maxResults) break;
504
- }
505
-
506
- return {
507
- events,
508
- checkpoint: {
509
- last_sync_at: new Date().toISOString(),
510
- } satisfies OutlookCheckpoint as Record<string, unknown>,
511
- };
491
+ return this.walkPages<GraphMessage>(ctx, http, {
492
+ window,
493
+ firstUrl,
494
+ maxResults: (config.max_results as number) ?? 50,
495
+ toEvent: (msg) => ({
496
+ origin_id: `outlook_msg_${msg.id}`,
497
+ title: msg.subject,
498
+ payload_text: msg.bodyPreview || msg.subject,
499
+ author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
500
+ source_url: msg.webLink,
501
+ occurred_at: new Date(msg.receivedDateTime),
502
+ origin_type: 'email',
503
+ metadata: {
504
+ from: msg.from?.emailAddress?.address,
505
+ to: formatRecipients(msg.toRecipients ?? []),
506
+ cc: formatRecipients(msg.ccRecipients ?? []),
507
+ importance: msg.importance,
508
+ has_attachments: msg.hasAttachments,
509
+ is_read: msg.isRead,
510
+ },
511
+ }),
512
+ // The next window starts where this one ended.
513
+ completed: { last_sync_at: window.end },
514
+ });
512
515
  }
513
516
 
514
517
  // -------------------------------------------------------------------------
@@ -517,68 +520,106 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
517
520
 
518
521
  private async syncCalendar(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
519
522
  const config = ctx.config as Record<string, unknown>;
520
- const lookbackDays = (config.lookback_days as number) ?? 7;
521
- const lookaheadDays = (config.lookahead_days as number) ?? 30;
522
- const maxResults = (config.max_results as number) ?? 100;
523
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
523
524
 
524
- const startDate = new Date();
525
- startDate.setDate(startDate.getDate() - lookbackDays);
526
- const endDate = new Date();
527
- endDate.setDate(endDate.getDate() + lookaheadDays);
528
-
529
- const events: EventEnvelope[] = [];
525
+ // A calendar view is a snapshot of a moving range, re-walked in full on
526
+ // every traversal; a capped walk resumes the same range next run.
527
+ const window = checkpoint.pending?.window ?? {
528
+ start: daysFromNow(-((config.lookback_days as number) ?? 7)),
529
+ end: daysFromNow((config.lookahead_days as number) ?? 30),
530
+ };
530
531
  const firstUrl =
531
532
  `${this.API_BASE}/me/calendarView` +
532
- `?startDateTime=${startDate.toISOString()}` +
533
- `&endDateTime=${endDate.toISOString()}` +
534
- `&$top=${Math.min(maxResults, this.PAGE_SIZE)}` +
533
+ `?startDateTime=${window.start}` +
534
+ `&endDateTime=${window.end}` +
535
+ `&$top=${this.PAGE_SIZE}` +
535
536
  '&$orderby=start/dateTime' +
536
537
  '&$select=id,subject,bodyPreview,organizer,attendees,start,end,location,isAllDay,isCancelled,webLink,createdDateTime';
537
538
 
539
+ return this.walkPages<GraphEvent>(ctx, http, {
540
+ window,
541
+ firstUrl,
542
+ maxResults: (config.max_results as number) ?? 100,
543
+ toEvent: (evt) => ({
544
+ origin_id: `outlook_evt_${evt.id}`,
545
+ title: evt.subject,
546
+ payload_text: evt.bodyPreview || evt.subject,
547
+ author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
548
+ source_url: evt.webLink,
549
+ occurred_at: new Date(evt.start.dateTime),
550
+ origin_type: 'calendar_event',
551
+ metadata: {
552
+ organizer: evt.organizer?.emailAddress?.address,
553
+ location: evt.location?.displayName,
554
+ attendee_count: evt.attendees?.length ?? 0,
555
+ is_all_day: evt.isAllDay,
556
+ is_cancelled: evt.isCancelled,
557
+ start_time: evt.start.dateTime,
558
+ end_time: evt.end.dateTime,
559
+ },
560
+ }),
561
+ completed: { last_sync_at: new Date().toISOString() },
562
+ });
563
+ }
564
+
565
+ /**
566
+ * Walk a Graph collection page by page, committing each page with the
567
+ * cursor of the next. `max_results` caps how much a run starts, never how
568
+ * much of a fetched page is stored: a page is committed whole, and the run
569
+ * stops between pages once the cap is reached, handing back `more`.
570
+ */
571
+ private async walkPages<T>(
572
+ ctx: SyncContext,
573
+ http: HttpClient,
574
+ walk: {
575
+ window: { start: string; end: string };
576
+ firstUrl: string;
577
+ maxResults: number;
578
+ toEvent: (item: T) => EventEnvelope;
579
+ completed: OutlookCheckpoint;
580
+ }
581
+ ): Promise<SyncResult> {
582
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
583
+ let url = checkpoint.pending?.next_link ?? walk.firstUrl;
584
+ // A saved link outlives the run that issued it and Graph may retire it.
585
+ // The first request of the run — the only one using a stored cursor — may
586
+ // restart the window from its first page instead of wedging the feed;
587
+ // `origin_id` makes that replay a supersede, not a duplicate.
588
+ let staleCursorRecoverable = checkpoint.pending !== undefined;
538
589
  let fetched = 0;
539
590
 
540
- const pages = paginateByCursor<GraphEvent, string>(
541
- async (url) => {
542
- const data = await http.get<GraphPagedResponse<GraphEvent>>(url ?? firstUrl);
543
- return { items: data.value, nextCursor: data['@odata.nextLink'] };
544
- },
545
- { maxPages: this.MAX_PAGES, initialCursor: firstUrl }
546
- );
547
-
548
- for await (const value of pages) {
549
- for (const evt of value) {
550
- if (fetched >= maxResults) break;
551
- events.push({
552
- origin_id: `outlook_evt_${evt.id}`,
553
- title: evt.subject,
554
- payload_text: evt.bodyPreview || evt.subject,
555
- author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
556
- source_url: evt.webLink,
557
- occurred_at: new Date(evt.start.dateTime),
558
- origin_type: 'calendar_event',
559
- metadata: {
560
- organizer: evt.organizer?.emailAddress?.address,
561
- location: evt.location?.displayName,
562
- attendee_count: evt.attendees?.length ?? 0,
563
- is_all_day: evt.isAllDay,
564
- is_cancelled: evt.isCancelled,
565
- start_time: evt.start.dateTime,
566
- end_time: evt.end.dateTime,
567
- },
568
- });
569
- fetched++;
591
+ for (let page = 0; page < this.MAX_PAGES; page++) {
592
+ let data: GraphPagedResponse<T>;
593
+ try {
594
+ data = await http.get<GraphPagedResponse<T>>(url);
595
+ } catch (error) {
596
+ if (staleCursorRecoverable && error instanceof HttpStatusError && [400, 404, 410].includes(error.status)) {
597
+ staleCursorRecoverable = false;
598
+ url = walk.firstUrl;
599
+ page--;
600
+ continue;
601
+ }
602
+ throw error;
570
603
  }
571
-
572
- if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
573
-
574
- if (fetched >= maxResults) break;
604
+ staleCursorRecoverable = false;
605
+
606
+ const events = data.value.map(walk.toEvent);
607
+ fetched += events.length;
608
+ const nextLink = data['@odata.nextLink'];
609
+ if (!nextLink) {
610
+ await ctx.commit(events, walk.completed as Record<string, unknown>);
611
+ return { status: 'complete', metadata: { items_found: fetched } };
612
+ }
613
+ const pending: OutlookCheckpoint = {
614
+ ...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
615
+ pending: { window: walk.window, next_link: nextLink },
616
+ };
617
+ await ctx.commit(events, pending as Record<string, unknown>);
618
+ url = nextLink;
619
+ if (fetched >= walk.maxResults) break;
575
620
  }
576
-
577
- return {
578
- events,
579
- checkpoint: {
580
- last_sync_at: new Date().toISOString(),
581
- } satisfies OutlookCheckpoint as Record<string, unknown>,
582
- };
621
+ // Pages remain: the committed cursor resumes this window next run.
622
+ return { status: 'more', metadata: { items_found: fetched } };
583
623
  }
624
+
584
625
  }
@@ -344,6 +344,9 @@ function toDate(v: unknown): Date | null {
344
344
  return null;
345
345
  }
346
346
 
347
+ /** Rows per `ctx.commit` during a sync. */
348
+ const COMMIT_ROWS = 500;
349
+
347
350
  export default class PostgresConnector extends ConnectorRuntime {
348
351
  readonly definition: RuntimeConnectorDefinition = {
349
352
  key: 'postgres',
@@ -466,21 +469,26 @@ export default class PostgresConnector extends ConnectorRuntime {
466
469
  // to feedKey only for direct/programmatic sync calls (no feedId).
467
470
  const originPrefix = ctx.feedId != null ? String(ctx.feedId) : ctx.feedKey;
468
471
 
469
- // Map rows → events, advancing the compound checkpoint to the last row.
470
- const events: EventEnvelope[] = [];
471
- let newCheckpoint: PgCheckpoint = checkpoint;
472
- for (const row of rows) {
473
- events.push(this.rowToEvent(originPrefix, row, config, cursorCol, pkCol));
474
- newCheckpoint = {
475
- last_cursor: toCheckpointValue(row[cursorCol]),
476
- last_pk: toCheckpointValue(row[pkCol]),
472
+ // Map rows → events and commit them in slices, each with the keyset
473
+ // checkpoint of its last row: that checkpoint resumes exactly after it,
474
+ // so a run cut short keeps every slice it committed.
475
+ for (let start = 0; start < rows.length; start += COMMIT_ROWS) {
476
+ const slice = rows.slice(start, start + COMMIT_ROWS);
477
+ const last = slice[slice.length - 1];
478
+ const sliceCheckpoint: PgCheckpoint = {
479
+ last_cursor: toCheckpointValue(last[cursorCol]),
480
+ last_pk: toCheckpointValue(last[pkCol]),
477
481
  };
482
+ await ctx.commit(
483
+ slice.map((row) => this.rowToEvent(originPrefix, row, config, cursorCol, pkCol)),
484
+ sliceCheckpoint as unknown as Record<string, unknown>
485
+ );
478
486
  }
479
487
 
480
488
  return {
481
- events,
482
- checkpoint: newCheckpoint as unknown as Record<string, unknown>,
483
- metadata: { items_found: events.length },
489
+ // A full page means rows remain past the cursor: run again now.
490
+ status: rows.length >= limit ? 'more' : 'complete',
491
+ metadata: { items_found: rows.length },
484
492
  };
485
493
  } finally {
486
494
  await sql.end({ timeout: 5 });
@@ -265,10 +265,12 @@ export default class ProductHuntConnector extends ConnectorRuntime {
265
265
 
266
266
  let result: ProductHuntPostsResponse;
267
267
  try {
268
- result = await http.post<ProductHuntPostsResponse>(this.API_URL, {
269
- query: LIST_POSTS_QUERY,
270
- variables,
271
- });
268
+ // A GraphQL query sent as POST: safe to repeat on a 5xx.
269
+ result = await http.post<ProductHuntPostsResponse>(
270
+ this.API_URL,
271
+ { query: LIST_POSTS_QUERY, variables },
272
+ { idempotent: true }
273
+ );
272
274
  } catch (error) {
273
275
  if (error instanceof HttpStatusError && error.status === 401) {
274
276
  // PH v2 API requires auth. Fail the run instead of returning an
@@ -352,9 +354,9 @@ export default class ProductHuntConnector extends ConnectorRuntime {
352
354
  last_sync_at: new Date().toISOString(),
353
355
  };
354
356
 
357
+ await ctx.commit(events, checkpoint as Record<string, unknown>);
355
358
  return {
356
- events,
357
- checkpoint: checkpoint as Record<string, unknown>,
359
+ status: 'complete',
358
360
  metadata: {
359
361
  items_found: events.length,
360
362
  },
@@ -362,9 +362,9 @@ export default class RedditConnector extends ConnectorRuntime {
362
362
  pagination_token: after ?? undefined,
363
363
  };
364
364
 
365
+ await ctx.commit(events, checkpoint as Record<string, unknown>);
365
366
  return {
366
- events,
367
- checkpoint: checkpoint as Record<string, unknown>,
367
+ status: 'complete',
368
368
  metadata: {
369
369
  items_found: events.length,
370
370
  },
@@ -181,9 +181,9 @@ export default class RSSConnector extends ConnectorRuntime {
181
181
  last_published_at: latestPublishedAt,
182
182
  };
183
183
 
184
+ await ctx.commit(events, newCheckpoint as unknown as Record<string, unknown>);
184
185
  return {
185
- events,
186
- checkpoint: newCheckpoint as unknown as Record<string, unknown>,
186
+ status: 'complete',
187
187
  metadata: {
188
188
  items_found: events.length,
189
189
  feeds_fetched: feedUrls.length - fetchErrors.length,
@@ -143,6 +143,25 @@ export const slackChatUserIdentity: ChatUserIdentity = {
143
143
  buildUserKey(teamId, platformUserId) {
144
144
  return normalizeSlackUserId(teamId, platformUserId);
145
145
  },
146
+ userKeyScope(teamId) {
147
+ // No usable team = REFUSE, never "search the whole namespace". Two
148
+ // workspaces can both hold `U12345`, so an unconstrained search here would
149
+ // hand back an id from the wrong one. Validate exactly as
150
+ // `normalizeSlackUserId` does so the prefix cannot disagree with the keys
151
+ // it wrote.
152
+ if (typeof teamId !== 'string') return null;
153
+ const t = teamId.trim();
154
+ if (!t || !/^[a-z0-9_-]+$/i.test(t)) return null;
155
+ return { kind: 'team-prefix', prefix: `${t.toUpperCase()}:` };
156
+ },
157
+ // Slack addresses a user by the BARE `U…`; the team prefix is storage scoping
158
+ // only and would not resolve if sent back. Re-validating through
159
+ // `normalizeTeamScopedId` first means a key this connector could not have
160
+ // written (no prefix, bad chars) yields null rather than a half-parsed id.
161
+ platformUserIdFromKey(key) {
162
+ const normalized = normalizeTeamScopedId(key);
163
+ return normalized ? normalized.slice(normalized.indexOf(':') + 1) : null;
164
+ },
146
165
  };
147
166
 
148
167
  /** A Slack channel and the (bare `U…`) ids of its members. */
@@ -937,7 +937,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
937
937
 
938
938
  private async syncMessages(
939
939
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>
940
- ): Promise<SyncResult<BrowserCheckpoint>> {
940
+ ): Promise<SyncResult> {
941
941
  const dispatcher = requireExtensionDispatcher(ctx);
942
942
  const tabId = await readyWhatsAppTab(dispatcher);
943
943
  return await this.collectMessages(ctx, dispatcher, tabId);
@@ -954,7 +954,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
954
954
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
955
955
  dispatcher: ChromeActionDispatcher,
956
956
  tabId: number
957
- ): Promise<SyncResult<BrowserCheckpoint>> {
957
+ ): Promise<SyncResult> {
958
958
  try {
959
959
  return await this.collectMessagesInner(ctx, dispatcher, tabId);
960
960
  } catch (error) {
@@ -968,7 +968,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
968
968
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
969
969
  dispatcher: ChromeActionDispatcher,
970
970
  tabId: number
971
- ): Promise<SyncResult<BrowserCheckpoint>> {
971
+ ): Promise<SyncResult> {
972
972
  const { checkpoint, request } = buildCollectionPlan({
973
973
  checkpoint: ctx.checkpoint as Record<string, unknown> | null,
974
974
  config: ctx.config as Record<string, unknown>,
@@ -1144,7 +1144,8 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
1144
1144
  nextCheckpoint.diagnostics =
1145
1145
  Object.keys(diagnostics).length > 0 ? diagnostics : undefined;
1146
1146
 
1147
- return { events, checkpoint: nextCheckpoint };
1147
+ await ctx.commit(events, nextCheckpoint as unknown as BrowserCheckpoint);
1148
+ return { status: "complete" };
1148
1149
  }
1149
1150
 
1150
1151
  async execute(ctx: ActionContext): Promise<ActionResult> {
@@ -47,6 +47,13 @@ import {
47
47
  type SyncContext,
48
48
  type SyncResult,
49
49
  } from "@lobu/connector-sdk";
50
+
51
+ /** What one X sync gathers before it is committed. */
52
+ type XSyncPage = {
53
+ events: EventEnvelope[];
54
+ checkpoint: Record<string, unknown>;
55
+ metadata?: SyncResult["metadata"];
56
+ };
50
57
  import { X_IDENTITY, normalizeXHandle } from "./x-identity.js";
51
58
 
52
59
  /** OAuth scopes needed per feed for the API path (not used to pause browser-capable feeds). */
@@ -1061,7 +1068,7 @@ export function finalizeDmSyncResult(
1061
1068
  messages: XDmMessage[],
1062
1069
  checkpoint: XCheckpoint,
1063
1070
  metadata: Record<string, unknown>,
1064
- ): SyncResult {
1071
+ ): XSyncPage {
1065
1072
  const seenIds = new Set<string>();
1066
1073
  const deduped = messages.filter((message) => {
1067
1074
  if (!message.id || !message.text || seenIds.has(message.id)) return false;
@@ -1104,7 +1111,7 @@ export function finalizeSyncResult(
1104
1111
  checkpoint: XCheckpoint,
1105
1112
  metadata: Record<string, unknown>,
1106
1113
  options?: { originType?: string },
1107
- ): SyncResult {
1114
+ ): XSyncPage {
1108
1115
  const seenIds = new Set<string>();
1109
1116
  const deduped = tweets.filter((tweet) => {
1110
1117
  if (!tweet.id || !tweet.text || seenIds.has(tweet.id)) return false;
@@ -1151,7 +1158,7 @@ export function finalizeLikedTweetsResult(
1151
1158
  pagesRead: number;
1152
1159
  historicalItemsRead: number;
1153
1160
  },
1154
- ): SyncResult {
1161
+ ): XSyncPage {
1155
1162
  // A browser backfill intentionally re-observes the newest page on every run.
1156
1163
  // Do not drop the old boundary: deterministic origin ids let ingestion update
1157
1164
  // metrics/media without creating a second canonical event.
@@ -1498,7 +1505,7 @@ function shouldFallbackToExtension(error: unknown): boolean {
1498
1505
  return isOAuthScopeOrAuthError(error) || isOAuthLookupFallbackError(error);
1499
1506
  }
1500
1507
 
1501
- async function syncWithOAuthFallback<T extends SyncResult>(
1508
+ async function syncWithOAuthFallback<T extends XSyncPage>(
1502
1509
  oauthFn: () => Promise<T>,
1503
1510
  extensionFn: () => Promise<T>,
1504
1511
  ): Promise<T> {
@@ -1511,7 +1518,7 @@ async function syncWithOAuthFallback<T extends SyncResult>(
1511
1518
  }
1512
1519
 
1513
1520
  /** Browser-first feeds fall back to the extension unless `use_oauth` is set. */
1514
- async function syncOAuthWithOptionalFallback<T extends SyncResult>(
1521
+ async function syncOAuthWithOptionalFallback<T extends XSyncPage>(
1515
1522
  config: Record<string, unknown>,
1516
1523
  oauthFn: () => Promise<T>,
1517
1524
  extensionFn: () => Promise<T>,
@@ -1774,7 +1781,7 @@ async function syncViaOAuthApi(
1774
1781
  ctx: SyncContext,
1775
1782
  config: Record<string, unknown>,
1776
1783
  checkpoint: XCheckpoint,
1777
- ): Promise<SyncResult> {
1784
+ ): Promise<XSyncPage> {
1778
1785
  const accessToken = ctx.credentials?.accessToken;
1779
1786
  if (!accessToken) {
1780
1787
  throw new Error("OAuth access token missing for X connector");
@@ -1876,7 +1883,7 @@ async function syncViaExtension(args: {
1876
1883
  /** Extra metadata to fold into the result (e.g. which timeline tab). */
1877
1884
  metadata?: Record<string, unknown>;
1878
1885
  originType?: string;
1879
- }): Promise<SyncResult> {
1886
+ }): Promise<XSyncPage> {
1880
1887
  const { ctx, url, interceptPatterns, parseResponse, maxScrolls, checkpoint } =
1881
1888
  args;
1882
1889
  const result = await extensionNetworkSync<XTweet>({
@@ -1909,7 +1916,7 @@ async function syncSearchViaExtension(
1909
1916
  ctx: SyncContext,
1910
1917
  config: Record<string, unknown>,
1911
1918
  checkpoint: XCheckpoint,
1912
- ): Promise<SyncResult> {
1919
+ ): Promise<XSyncPage> {
1913
1920
  const searchQuery = buildSearchQuery(config);
1914
1921
  const maxScrolls = readMaxPages(config);
1915
1922
  const searchFilter = (config.search_filter as string) ?? "live";
@@ -1930,7 +1937,7 @@ async function syncMyTweetsViaOAuthApi(
1930
1937
  ctx: SyncContext,
1931
1938
  config: Record<string, unknown>,
1932
1939
  checkpoint: XCheckpoint,
1933
- ): Promise<SyncResult> {
1940
+ ): Promise<XSyncPage> {
1934
1941
  const accessToken = ctx.credentials?.accessToken;
1935
1942
  if (!accessToken) {
1936
1943
  throw new Error("OAuth access token missing for my_tweets feed");
@@ -1977,7 +1984,7 @@ async function syncMyTweetsViaExtension(
1977
1984
  ctx: SyncContext,
1978
1985
  config: Record<string, unknown>,
1979
1986
  checkpoint: XCheckpoint,
1980
- ): Promise<SyncResult> {
1987
+ ): Promise<XSyncPage> {
1981
1988
  const accountHandle = await resolveAccountHandle(config);
1982
1989
  const maxScrolls = readMaxPages(config);
1983
1990
  const profileUrl = `https://x.com/${encodeURIComponent(accountHandle)}`;
@@ -1997,7 +2004,7 @@ async function syncLikedTweetsViaExtension(
1997
2004
  ctx: SyncContext,
1998
2005
  config: Record<string, unknown>,
1999
2006
  checkpoint: XCheckpoint,
2000
- ): Promise<SyncResult> {
2007
+ ): Promise<XSyncPage> {
2001
2008
  const accountHandle = await resolveAccountHandle(config);
2002
2009
  const likesUrl = `https://x.com/${encodeURIComponent(accountHandle)}/likes`;
2003
2010
  const dispatcher = requireExtensionDispatcher(ctx);
@@ -2130,7 +2137,7 @@ async function syncLikedTweetsViaOAuthApi(
2130
2137
  ctx: SyncContext,
2131
2138
  config: Record<string, unknown>,
2132
2139
  checkpoint: XCheckpoint,
2133
- ): Promise<SyncResult> {
2140
+ ): Promise<XSyncPage> {
2134
2141
  const accessToken = ctx.credentials?.accessToken;
2135
2142
  if (!accessToken) {
2136
2143
  throw new Error("OAuth access token missing for liked_tweets feed");
@@ -2201,7 +2208,7 @@ async function syncBookmarksViaOAuthApi(
2201
2208
  ctx: SyncContext,
2202
2209
  config: Record<string, unknown>,
2203
2210
  checkpoint: XCheckpoint,
2204
- ): Promise<SyncResult> {
2211
+ ): Promise<XSyncPage> {
2205
2212
  const accessToken = ctx.credentials?.accessToken;
2206
2213
  if (!accessToken) {
2207
2214
  throw new Error("OAuth access token missing for bookmarks feed");
@@ -2240,7 +2247,7 @@ async function syncBookmarksViaExtension(
2240
2247
  ctx: SyncContext,
2241
2248
  config: Record<string, unknown>,
2242
2249
  checkpoint: XCheckpoint,
2243
- ): Promise<SyncResult> {
2250
+ ): Promise<XSyncPage> {
2244
2251
  const maxScrolls = readMaxPages(config);
2245
2252
 
2246
2253
  return syncViaExtension({
@@ -2278,7 +2285,7 @@ async function syncDirectMessagesViaOAuthApi(
2278
2285
  ctx: SyncContext,
2279
2286
  config: Record<string, unknown>,
2280
2287
  checkpoint: XCheckpoint,
2281
- ): Promise<SyncResult> {
2288
+ ): Promise<XSyncPage> {
2282
2289
  const accessToken = ctx.credentials?.accessToken;
2283
2290
  if (!accessToken) {
2284
2291
  throw new Error(
@@ -2332,7 +2339,7 @@ async function syncDirectMessagesViaExtension(
2332
2339
  ctx: SyncContext,
2333
2340
  config: Record<string, unknown>,
2334
2341
  checkpoint: XCheckpoint,
2335
- ): Promise<SyncResult> {
2342
+ ): Promise<XSyncPage> {
2336
2343
  const maxScrolls = readMaxPages(config);
2337
2344
  let authUserId =
2338
2345
  typeof config.account_user_id === "string"
@@ -2378,7 +2385,7 @@ async function syncHomeFeedViaDomScrape(
2378
2385
  ctx: SyncContext,
2379
2386
  config: Record<string, unknown>,
2380
2387
  checkpoint: XCheckpoint,
2381
- ): Promise<SyncResult> {
2388
+ ): Promise<XSyncPage> {
2382
2389
  const maxScrolls = readScrollBudget(config, { defaultMax: 10, cap: 30 });
2383
2390
  const { items: rows, loggedIn } = await extensionDomScrape<HomeFeedRow>({
2384
2391
  dispatcher: requireExtensionDispatcher(ctx),
@@ -3343,6 +3350,14 @@ export default class XConnector extends ConnectorRuntime {
3343
3350
  }
3344
3351
 
3345
3352
  private async syncFeed(ctx: SyncContext): Promise<SyncResult> {
3353
+ // Collected whole, then committed once: an OAuth attempt that fails part
3354
+ // way falls back to the extension, and nothing it gathered may land first.
3355
+ const page = await this.collectFeed(ctx);
3356
+ await ctx.commit(page.events, page.checkpoint);
3357
+ return { status: "complete", ...(page.metadata ? { metadata: page.metadata } : {}) };
3358
+ }
3359
+
3360
+ private async collectFeed(ctx: SyncContext): Promise<XSyncPage> {
3346
3361
  const config = ctx.config as Record<string, unknown>;
3347
3362
  const checkpoint = (ctx.checkpoint ?? {}) as XCheckpoint;
3348
3363
  const feedKey = ctx.feedKey ?? "tweets";