@lobu/runtime-device 20.2.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +42 -36
  3. package/npm-shrinkwrap.json +270 -224
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/config/define.d.ts +25 -8
  31. package/vendor/cli/dist/config/define.js +24 -0
  32. package/vendor/cli/dist/connectors/README.md +15 -16
  33. package/vendor/cli/dist/connectors/github.ts +15 -9
  34. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  35. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  36. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  37. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  38. package/vendor/cli/dist/connectors/jira.ts +62 -27
  39. package/vendor/cli/dist/connectors/linear.ts +55 -21
  40. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  41. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  42. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  43. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  44. package/vendor/cli/dist/connectors/rss.ts +2 -2
  45. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  46. package/vendor/cli/dist/connectors/x.ts +32 -17
  47. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  48. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  49. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  50. package/vendor/cli/dist/runtime-components.json +4 -4
  51. package/vendor/cli/package.json +2 -1
  52. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  53. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  54. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  55. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  56. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  57. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  58. package/vendor/connector-sdk/dist/http-client.js +15 -13
  59. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  60. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  61. package/vendor/connector-sdk/dist/retry.js +40 -64
  62. package/vendor/connector-sdk/package.json +1 -1
  63. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  64. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  65. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  66. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  67. package/vendor/connector-worker/dist/compile/index.js +41 -8
  68. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  69. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  70. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  71. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  72. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  73. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  74. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  75. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  76. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  77. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  78. package/vendor/connector-worker/package.json +1 -1
  79. package/vendor/core/dist/agent-policy.js +3 -0
  80. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  81. package/vendor/core/dist/connector-query-errors.js +8 -3
  82. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  83. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  84. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  85. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  86. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  87. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  88. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  89. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  90. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  91. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  92. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  93. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  94. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  95. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  96. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  97. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  98. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  99. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  100. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  101. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  102. package/vendor/core/dist/reserved.d.ts +3 -3
  103. package/vendor/core/dist/reserved.js +9 -1
  104. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  105. package/vendor/core/package.json +25 -5
  106. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  107. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -12,7 +12,7 @@ import {
12
12
  type EventEnvelope,
13
13
  type FeedReadContext,
14
14
  type FeedReadResult,
15
- paginateByCursor,
15
+ HttpStatusError,
16
16
  requireBearerClient,
17
17
  type RuntimeConnectorDefinition,
18
18
  type SyncContext,
@@ -64,7 +64,15 @@ interface JiraConfig {
64
64
  }
65
65
 
66
66
  interface JiraCheckpoint {
67
+ /** When the last complete traversal began; the next one fetches issues updated since. */
67
68
  last_sync_at?: string;
69
+ /**
70
+ * A traversal that stopped at the per-run page cap: the fixed `updated`
71
+ * window it walks and the `nextPageToken` of the first page not yet stored,
72
+ * so the next run continues instead of re-reading the newest issues and
73
+ * never reaching the rest.
74
+ */
75
+ pending?: { window: { start: string; end: string }; page_token: string };
68
76
  }
69
77
 
70
78
  interface JiraUser {
@@ -519,52 +527,79 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
519
527
  // Sync into local memory.
520
528
  // -------------------------------------------------------------------------
521
529
 
522
- private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult<JiraCheckpoint>> {
530
+ private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult> {
523
531
  const base = await this.restBase(ctx.config, ctx.sessionState, ctx.credentials);
524
532
  const http = this.client(ctx.credentials);
533
+ const checkpoint = ctx.checkpoint ?? {};
525
534
  const lookbackDays = ctx.config.lookback_days ?? 365;
535
+ // Fixed per traversal: issues updated since the last complete traversal
536
+ // began, up to when this one began. An issue edited while the walk runs
537
+ // falls in the next window, and `origin_id` makes a re-report a supersede.
538
+ const window = checkpoint.pending?.window ?? {
539
+ start: checkpoint.last_sync_at ?? new Date(0).toISOString(),
540
+ end: new Date().toISOString(),
541
+ };
526
542
  const jql = buildJiraJql({
527
543
  baseQuery: asString(ctx.config.query) ?? asString(ctx.config.jql) ?? '',
528
544
  defaultWhenEmpty: `updated >= -${lookbackDays}d`,
545
+ window,
529
546
  });
530
547
 
531
- const events: EventEnvelope[] = [];
532
-
533
548
  // `/rest/api/3/search` was removed by Atlassian (CHANGE-2046); the
534
549
  // replacement `/search/jql` paginates with an opaque nextPageToken and
535
550
  // returns no total — iterate until the token is absent.
536
- const pages = paginateByCursor<JiraIssue, string>(
537
- async (nextPageToken) => {
538
- const params = new URLSearchParams({
539
- jql,
540
- maxResults: String(this.PAGE_SIZE),
541
- fields: ISSUE_FIELDS,
542
- });
543
- if (nextPageToken) params.set('nextPageToken', nextPageToken);
544
- const data = await http.json<JiraSearchResponse>(
551
+ let pageToken = checkpoint.pending?.page_token;
552
+ // A stored token may be retired between runs. The first request of the run
553
+ // — the only one using a token from a previous run — may restart the same
554
+ // window from its first page instead of wedging the feed.
555
+ let staleCursorRecoverable = pageToken !== undefined;
556
+ let found = 0;
557
+ for (let page = 0; page < this.MAX_PAGES; page++) {
558
+ const params = new URLSearchParams({
559
+ jql,
560
+ maxResults: String(this.PAGE_SIZE),
561
+ fields: ISSUE_FIELDS,
562
+ });
563
+ if (pageToken) params.set('nextPageToken', pageToken);
564
+ let data: JiraSearchResponse;
565
+ try {
566
+ data = await http.json<JiraSearchResponse>(
545
567
  `${base}/search/jql?${params.toString()}`,
546
568
  { method: 'GET', headers: { Accept: 'application/json' } },
547
569
  );
548
- return { items: data.issues ?? [], nextCursor: data.nextPageToken };
549
- },
550
- { maxPages: this.MAX_PAGES },
551
- );
570
+ } catch (error) {
571
+ if (staleCursorRecoverable && error instanceof HttpStatusError && error.status === 400) {
572
+ staleCursorRecoverable = false;
573
+ pageToken = undefined;
574
+ page--;
575
+ continue;
576
+ }
577
+ throw error;
578
+ }
579
+ staleCursorRecoverable = false;
552
580
 
553
- for await (const issues of pages) {
581
+ const issues = data.issues ?? [];
582
+ const events: EventEnvelope[] = [];
554
583
  for (const issue of issues) {
555
584
  const event = this.issueEvent(issue, ctx.config);
556
585
  if (event) events.push(event);
557
586
  }
558
- // Preserve the original early-exit on an empty page even when a token is
559
- // returned — guards against a degenerate self-referential cursor.
560
- if (issues.length === 0) break;
561
- }
587
+ found += events.length;
562
588
 
563
- return {
564
- events,
565
- checkpoint: { last_sync_at: new Date().toISOString() },
566
- metadata: { items_found: events.length },
567
- };
589
+ // An empty page ends the walk even when a token is returned — guards
590
+ // against a degenerate self-referential cursor.
591
+ if (!data.nextPageToken || issues.length === 0) {
592
+ await ctx.commit(events, { last_sync_at: window.end });
593
+ return { status: 'complete', metadata: { items_found: found } };
594
+ }
595
+ pageToken = data.nextPageToken;
596
+ await ctx.commit(events, {
597
+ ...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
598
+ pending: { window, page_token: pageToken },
599
+ });
600
+ }
601
+ // The page cap stopped the walk; its committed token resumes it next run.
602
+ return { status: 'more', metadata: { items_found: found } };
568
603
  }
569
604
 
570
605
  // -------------------------------------------------------------------------
@@ -40,7 +40,15 @@ interface LinearConfig {
40
40
  }
41
41
 
42
42
  interface LinearCheckpoint {
43
+ /** When the last complete traversal began; the next one fetches issues updated since. */
43
44
  last_sync_at?: string;
45
+ /**
46
+ * A traversal that stopped at the per-run page cap. `since`/`started_at`
47
+ * are the fixed bounds it walks and `cursor` the `endCursor` of the last page
48
+ * stored, so the next run continues past it instead of re-reading the newest
49
+ * issues and never reaching the rest.
50
+ */
51
+ pending?: { since?: string; started_at: string; cursor: string };
44
52
  }
45
53
 
46
54
  interface LinearUser {
@@ -297,7 +305,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
297
305
  pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
298
306
  nodes?: LinearIssueNode[];
299
307
  };
300
- } = await this.graphql(ctx.credentials, gql);
308
+ } = await this.graphql(ctx.credentials, gql, undefined, { idempotent: true });
301
309
 
302
310
  const rows = (response.issues?.nodes ?? [])
303
311
  .map((node) => this.issueRow(node))
@@ -323,16 +331,24 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
323
331
  // Sync into local memory.
324
332
  // -------------------------------------------------------------------------
325
333
 
326
- private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult<LinearCheckpoint>> {
327
- const events: EventEnvelope[] = [];
328
- let cursor: string | null = null;
329
- let pages = 0;
334
+ private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult> {
335
+ const checkpoint = ctx.checkpoint ?? {};
336
+ // Fixed per traversal: issues updated after the last complete traversal
337
+ // began (none on the first), walked until the listing ends. An issue edited
338
+ // while the walk runs is re-reported by the next traversal, and `origin_id`
339
+ // makes that a supersede, not a duplicate.
340
+ const since = checkpoint.pending ? checkpoint.pending.since : checkpoint.last_sync_at;
341
+ const startedAt = checkpoint.pending?.started_at ?? new Date().toISOString();
342
+ let cursor: string | null = checkpoint.pending?.cursor ?? null;
330
343
 
331
- const filter = ctx.config.team_key
332
- ? `, filter: { team: { key: { eq: ${JSON.stringify(ctx.config.team_key)} } } }`
333
- : '';
344
+ const filter = buildLinearIssueFilter({
345
+ teamKey: asString(ctx.config.team_key),
346
+ updatedAfterIso: since,
347
+ updatedBeforeIso: startedAt,
348
+ });
334
349
 
335
- while (pages < this.MAX_PAGES) {
350
+ let found = 0;
351
+ for (let pages = 0; pages < this.MAX_PAGES; pages++) {
336
352
  const after: string = cursor ? `, after: ${JSON.stringify(cursor)}` : '';
337
353
  const query: string = `
338
354
  query {
@@ -348,25 +364,36 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
348
364
  pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
349
365
  nodes?: LinearIssueNode[];
350
366
  };
351
- }>(ctx.credentials, query);
367
+ }>(ctx.credentials, query, undefined, { idempotent: true });
352
368
 
353
- const nodes = response.issues?.nodes ?? [];
369
+ const nodes = response.issues?.nodes;
370
+ const pageInfo = response.issues?.pageInfo;
371
+ if (
372
+ !Array.isArray(nodes) ||
373
+ typeof pageInfo?.hasNextPage !== 'boolean' ||
374
+ (pageInfo.hasNextPage && !pageInfo.endCursor)
375
+ ) {
376
+ throw new Error('Linear did not return a valid page cursor/exhaustion state.');
377
+ }
378
+ const events: EventEnvelope[] = [];
354
379
  for (const node of nodes) {
355
380
  const event = this.issueEvent(node);
356
381
  if (event) events.push(event);
357
382
  }
383
+ found += events.length;
358
384
 
359
- pages += 1;
360
- const pageInfo = response.issues?.pageInfo;
361
- if (!pageInfo?.hasNextPage || !pageInfo.endCursor) break;
385
+ if (!pageInfo?.hasNextPage || !pageInfo.endCursor) {
386
+ await ctx.commit(events, { last_sync_at: startedAt });
387
+ return { status: 'complete', metadata: { items_found: found } };
388
+ }
362
389
  cursor = pageInfo.endCursor;
390
+ await ctx.commit(events, {
391
+ ...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
392
+ pending: { ...(since ? { since } : {}), started_at: startedAt, cursor },
393
+ });
363
394
  }
364
-
365
- return {
366
- events,
367
- checkpoint: { last_sync_at: new Date().toISOString() },
368
- metadata: { items_found: events.length },
369
- };
395
+ // The page cap stopped the walk; its committed cursor resumes it next run.
396
+ return { status: 'more', metadata: { items_found: found } };
370
397
  }
371
398
 
372
399
  // -------------------------------------------------------------------------
@@ -463,10 +490,16 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
463
490
  // GraphQL transport
464
491
  // -------------------------------------------------------------------------
465
492
 
493
+ /**
494
+ * `idempotent` marks a read: a GraphQL query goes out as POST, so without it
495
+ * the client never repeats the request on a 5xx. Leave it off for mutations —
496
+ * a 5xx does not say whether the webhook was already created.
497
+ */
466
498
  private async graphql<T>(
467
499
  credentials: SyncCredentials | null,
468
500
  query: string,
469
- variables?: Record<string, unknown>
501
+ variables?: Record<string, unknown>,
502
+ options: { idempotent?: boolean } = {}
470
503
  ): Promise<T> {
471
504
  const http = requireBearerClient(credentials, {
472
505
  errorPrefix: 'Linear API',
@@ -478,6 +511,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
478
511
  method: 'POST',
479
512
  headers: { 'Content-Type': 'application/json' },
480
513
  body: JSON.stringify({ query, variables: variables ?? {} }),
514
+ idempotent: options.idempotent,
481
515
  }
482
516
  );
483
517
 
@@ -14,9 +14,9 @@ import {
14
14
  type EventEnvelope,
15
15
  fileDownloadOutput,
16
16
  type HttpClient,
17
+ HttpStatusError,
17
18
  inlineContentBudget,
18
19
  inlineMaxBytesSchema,
19
- paginateByCursor,
20
20
  requireBearerClient,
21
21
  type SyncContext,
22
22
  type SyncResult,
@@ -89,7 +89,24 @@ interface GraphPagedResponse<T> {
89
89
  // ---------------------------------------------------------------------------
90
90
 
91
91
  interface OutlookCheckpoint {
92
+ /** When the last complete traversal began; the next window starts here. */
92
93
  last_sync_at?: string;
94
+ /**
95
+ * A traversal that stopped at the per-run cap with pages left. `window`
96
+ * is the fixed range it walks and `next_link` Graph's cursor for the first
97
+ * page not yet stored, so the next run continues exactly where this one
98
+ * stopped instead of re-reading the newest page and never reaching the rest.
99
+ */
100
+ pending?: { window: { start: string; end: string }; next_link: string };
101
+ }
102
+
103
+ const WINDOW_OVERLAP_MS = 5 * 60_000;
104
+
105
+ /** ISO timestamp `days` from now (negative for the past). */
106
+ function daysFromNow(days: number): string {
107
+ const date = new Date();
108
+ date.setDate(date.getDate() + days);
109
+ return date.toISOString();
93
110
  }
94
111
 
95
112
  // ---------------------------------------------------------------------------
@@ -448,67 +465,53 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
448
465
  private async syncMessages(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
449
466
  const config = ctx.config as Record<string, unknown>;
450
467
  const folder = (config.folder as string) ?? 'inbox';
451
- const maxResults = (config.max_results as number) ?? 50;
452
468
  const lookbackDays = (config.lookback_days as number) ?? 30;
453
- const encodedFolder = encodeURIComponent(folder);
454
-
455
- const since = new Date();
456
- since.setDate(since.getDate() - lookbackDays);
457
- const sinceFilter = since.toISOString();
458
-
459
- const events: EventEnvelope[] = [];
469
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
470
+
471
+ // A fixed window per traversal: from where the last complete one began
472
+ // (or the lookback) up to when this one began. Mail arriving meanwhile
473
+ // belongs to the next window, so a capped walk can never be pushed past
474
+ // messages it has not reached by newer arrivals landing at the front.
475
+ // The start overlaps the previous window by a few minutes: Graph can
476
+ // surface a message after its receivedDateTime, and `origin_id` makes the
477
+ // replayed boundary a supersede, not a duplicate.
478
+ const window = checkpoint.pending?.window ?? {
479
+ start: checkpoint.last_sync_at
480
+ ? new Date(Date.parse(checkpoint.last_sync_at) - WINDOW_OVERLAP_MS).toISOString()
481
+ : daysFromNow(-lookbackDays),
482
+ end: new Date().toISOString(),
483
+ };
460
484
  const firstUrl =
461
- `${this.API_BASE}/me/mailFolders/${encodedFolder}/messages` +
462
- `?$top=${Math.min(maxResults, this.PAGE_SIZE)}` +
485
+ `${this.API_BASE}/me/mailFolders/${encodeURIComponent(folder)}/messages` +
486
+ `?$top=${this.PAGE_SIZE}` +
463
487
  '&$orderby=receivedDateTime desc' +
464
- `&$filter=receivedDateTime ge ${sinceFilter}` +
488
+ `&$filter=receivedDateTime ge ${window.start} and receivedDateTime lt ${window.end}` +
465
489
  '&$select=id,conversationId,subject,bodyPreview,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,importance,isRead,webLink';
466
490
 
467
- let fetched = 0;
468
-
469
- // Graph paginates via full `@odata.nextLink` URLs, so the cursor is the page URL.
470
- const pages = paginateByCursor<GraphMessage, string>(
471
- async (url) => {
472
- const data = await http.get<GraphPagedResponse<GraphMessage>>(url ?? firstUrl);
473
- return { items: data.value, nextCursor: data['@odata.nextLink'] };
474
- },
475
- { maxPages: this.MAX_PAGES, initialCursor: firstUrl }
476
- );
477
-
478
- for await (const value of pages) {
479
- for (const msg of value) {
480
- if (fetched >= maxResults) break;
481
- events.push({
482
- origin_id: `outlook_msg_${msg.id}`,
483
- title: msg.subject,
484
- payload_text: msg.bodyPreview || msg.subject,
485
- author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
486
- source_url: msg.webLink,
487
- occurred_at: new Date(msg.receivedDateTime),
488
- origin_type: 'email',
489
- metadata: {
490
- from: msg.from?.emailAddress?.address,
491
- to: formatRecipients(msg.toRecipients ?? []),
492
- cc: formatRecipients(msg.ccRecipients ?? []),
493
- importance: msg.importance,
494
- has_attachments: msg.hasAttachments,
495
- is_read: msg.isRead,
496
- },
497
- });
498
- fetched++;
499
- }
500
-
501
- if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
502
-
503
- if (fetched >= maxResults) break;
504
- }
505
-
506
- return {
507
- events,
508
- checkpoint: {
509
- last_sync_at: new Date().toISOString(),
510
- } satisfies OutlookCheckpoint as Record<string, unknown>,
511
- };
491
+ return this.walkPages<GraphMessage>(ctx, http, {
492
+ window,
493
+ firstUrl,
494
+ maxResults: (config.max_results as number) ?? 50,
495
+ toEvent: (msg) => ({
496
+ origin_id: `outlook_msg_${msg.id}`,
497
+ title: msg.subject,
498
+ payload_text: msg.bodyPreview || msg.subject,
499
+ author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
500
+ source_url: msg.webLink,
501
+ occurred_at: new Date(msg.receivedDateTime),
502
+ origin_type: 'email',
503
+ metadata: {
504
+ from: msg.from?.emailAddress?.address,
505
+ to: formatRecipients(msg.toRecipients ?? []),
506
+ cc: formatRecipients(msg.ccRecipients ?? []),
507
+ importance: msg.importance,
508
+ has_attachments: msg.hasAttachments,
509
+ is_read: msg.isRead,
510
+ },
511
+ }),
512
+ // The next window starts where this one ended.
513
+ completed: { last_sync_at: window.end },
514
+ });
512
515
  }
513
516
 
514
517
  // -------------------------------------------------------------------------
@@ -517,68 +520,106 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
517
520
 
518
521
  private async syncCalendar(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
519
522
  const config = ctx.config as Record<string, unknown>;
520
- const lookbackDays = (config.lookback_days as number) ?? 7;
521
- const lookaheadDays = (config.lookahead_days as number) ?? 30;
522
- const maxResults = (config.max_results as number) ?? 100;
523
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
523
524
 
524
- const startDate = new Date();
525
- startDate.setDate(startDate.getDate() - lookbackDays);
526
- const endDate = new Date();
527
- endDate.setDate(endDate.getDate() + lookaheadDays);
528
-
529
- const events: EventEnvelope[] = [];
525
+ // A calendar view is a snapshot of a moving range, re-walked in full on
526
+ // every traversal; a capped walk resumes the same range next run.
527
+ const window = checkpoint.pending?.window ?? {
528
+ start: daysFromNow(-((config.lookback_days as number) ?? 7)),
529
+ end: daysFromNow((config.lookahead_days as number) ?? 30),
530
+ };
530
531
  const firstUrl =
531
532
  `${this.API_BASE}/me/calendarView` +
532
- `?startDateTime=${startDate.toISOString()}` +
533
- `&endDateTime=${endDate.toISOString()}` +
534
- `&$top=${Math.min(maxResults, this.PAGE_SIZE)}` +
533
+ `?startDateTime=${window.start}` +
534
+ `&endDateTime=${window.end}` +
535
+ `&$top=${this.PAGE_SIZE}` +
535
536
  '&$orderby=start/dateTime' +
536
537
  '&$select=id,subject,bodyPreview,organizer,attendees,start,end,location,isAllDay,isCancelled,webLink,createdDateTime';
537
538
 
539
+ return this.walkPages<GraphEvent>(ctx, http, {
540
+ window,
541
+ firstUrl,
542
+ maxResults: (config.max_results as number) ?? 100,
543
+ toEvent: (evt) => ({
544
+ origin_id: `outlook_evt_${evt.id}`,
545
+ title: evt.subject,
546
+ payload_text: evt.bodyPreview || evt.subject,
547
+ author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
548
+ source_url: evt.webLink,
549
+ occurred_at: new Date(evt.start.dateTime),
550
+ origin_type: 'calendar_event',
551
+ metadata: {
552
+ organizer: evt.organizer?.emailAddress?.address,
553
+ location: evt.location?.displayName,
554
+ attendee_count: evt.attendees?.length ?? 0,
555
+ is_all_day: evt.isAllDay,
556
+ is_cancelled: evt.isCancelled,
557
+ start_time: evt.start.dateTime,
558
+ end_time: evt.end.dateTime,
559
+ },
560
+ }),
561
+ completed: { last_sync_at: new Date().toISOString() },
562
+ });
563
+ }
564
+
565
+ /**
566
+ * Walk a Graph collection page by page, committing each page with the
567
+ * cursor of the next. `max_results` caps how much a run starts, never how
568
+ * much of a fetched page is stored: a page is committed whole, and the run
569
+ * stops between pages once the cap is reached, handing back `more`.
570
+ */
571
+ private async walkPages<T>(
572
+ ctx: SyncContext,
573
+ http: HttpClient,
574
+ walk: {
575
+ window: { start: string; end: string };
576
+ firstUrl: string;
577
+ maxResults: number;
578
+ toEvent: (item: T) => EventEnvelope;
579
+ completed: OutlookCheckpoint;
580
+ }
581
+ ): Promise<SyncResult> {
582
+ const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
583
+ let url = checkpoint.pending?.next_link ?? walk.firstUrl;
584
+ // A saved link outlives the run that issued it and Graph may retire it.
585
+ // The first request of the run — the only one using a stored cursor — may
586
+ // restart the window from its first page instead of wedging the feed;
587
+ // `origin_id` makes that replay a supersede, not a duplicate.
588
+ let staleCursorRecoverable = checkpoint.pending !== undefined;
538
589
  let fetched = 0;
539
590
 
540
- const pages = paginateByCursor<GraphEvent, string>(
541
- async (url) => {
542
- const data = await http.get<GraphPagedResponse<GraphEvent>>(url ?? firstUrl);
543
- return { items: data.value, nextCursor: data['@odata.nextLink'] };
544
- },
545
- { maxPages: this.MAX_PAGES, initialCursor: firstUrl }
546
- );
547
-
548
- for await (const value of pages) {
549
- for (const evt of value) {
550
- if (fetched >= maxResults) break;
551
- events.push({
552
- origin_id: `outlook_evt_${evt.id}`,
553
- title: evt.subject,
554
- payload_text: evt.bodyPreview || evt.subject,
555
- author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
556
- source_url: evt.webLink,
557
- occurred_at: new Date(evt.start.dateTime),
558
- origin_type: 'calendar_event',
559
- metadata: {
560
- organizer: evt.organizer?.emailAddress?.address,
561
- location: evt.location?.displayName,
562
- attendee_count: evt.attendees?.length ?? 0,
563
- is_all_day: evt.isAllDay,
564
- is_cancelled: evt.isCancelled,
565
- start_time: evt.start.dateTime,
566
- end_time: evt.end.dateTime,
567
- },
568
- });
569
- fetched++;
591
+ for (let page = 0; page < this.MAX_PAGES; page++) {
592
+ let data: GraphPagedResponse<T>;
593
+ try {
594
+ data = await http.get<GraphPagedResponse<T>>(url);
595
+ } catch (error) {
596
+ if (staleCursorRecoverable && error instanceof HttpStatusError && [400, 404, 410].includes(error.status)) {
597
+ staleCursorRecoverable = false;
598
+ url = walk.firstUrl;
599
+ page--;
600
+ continue;
601
+ }
602
+ throw error;
570
603
  }
571
-
572
- if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
573
-
574
- if (fetched >= maxResults) break;
604
+ staleCursorRecoverable = false;
605
+
606
+ const events = data.value.map(walk.toEvent);
607
+ fetched += events.length;
608
+ const nextLink = data['@odata.nextLink'];
609
+ if (!nextLink) {
610
+ await ctx.commit(events, walk.completed as Record<string, unknown>);
611
+ return { status: 'complete', metadata: { items_found: fetched } };
612
+ }
613
+ const pending: OutlookCheckpoint = {
614
+ ...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
615
+ pending: { window: walk.window, next_link: nextLink },
616
+ };
617
+ await ctx.commit(events, pending as Record<string, unknown>);
618
+ url = nextLink;
619
+ if (fetched >= walk.maxResults) break;
575
620
  }
576
-
577
- return {
578
- events,
579
- checkpoint: {
580
- last_sync_at: new Date().toISOString(),
581
- } satisfies OutlookCheckpoint as Record<string, unknown>,
582
- };
621
+ // Pages remain: the committed cursor resumes this window next run.
622
+ return { status: 'more', metadata: { items_found: fetched } };
583
623
  }
624
+
584
625
  }
@@ -344,6 +344,9 @@ function toDate(v: unknown): Date | null {
344
344
  return null;
345
345
  }
346
346
 
347
+ /** Rows per `ctx.commit` during a sync. */
348
+ const COMMIT_ROWS = 500;
349
+
347
350
  export default class PostgresConnector extends ConnectorRuntime {
348
351
  readonly definition: RuntimeConnectorDefinition = {
349
352
  key: 'postgres',
@@ -466,21 +469,26 @@ export default class PostgresConnector extends ConnectorRuntime {
466
469
  // to feedKey only for direct/programmatic sync calls (no feedId).
467
470
  const originPrefix = ctx.feedId != null ? String(ctx.feedId) : ctx.feedKey;
468
471
 
469
- // Map rows → events, advancing the compound checkpoint to the last row.
470
- const events: EventEnvelope[] = [];
471
- let newCheckpoint: PgCheckpoint = checkpoint;
472
- for (const row of rows) {
473
- events.push(this.rowToEvent(originPrefix, row, config, cursorCol, pkCol));
474
- newCheckpoint = {
475
- last_cursor: toCheckpointValue(row[cursorCol]),
476
- last_pk: toCheckpointValue(row[pkCol]),
472
+ // Map rows → events and commit them in slices, each with the keyset
473
+ // checkpoint of its last row: that checkpoint resumes exactly after it,
474
+ // so a run cut short keeps every slice it committed.
475
+ for (let start = 0; start < rows.length; start += COMMIT_ROWS) {
476
+ const slice = rows.slice(start, start + COMMIT_ROWS);
477
+ const last = slice[slice.length - 1];
478
+ const sliceCheckpoint: PgCheckpoint = {
479
+ last_cursor: toCheckpointValue(last[cursorCol]),
480
+ last_pk: toCheckpointValue(last[pkCol]),
477
481
  };
482
+ await ctx.commit(
483
+ slice.map((row) => this.rowToEvent(originPrefix, row, config, cursorCol, pkCol)),
484
+ sliceCheckpoint as unknown as Record<string, unknown>
485
+ );
478
486
  }
479
487
 
480
488
  return {
481
- events,
482
- checkpoint: newCheckpoint as unknown as Record<string, unknown>,
483
- metadata: { items_found: events.length },
489
+ // A full page means rows remain past the cursor: run again now.
490
+ status: rows.length >= limit ? 'more' : 'complete',
491
+ metadata: { items_found: rows.length },
484
492
  };
485
493
  } finally {
486
494
  await sql.end({ timeout: 5 });
@@ -265,10 +265,12 @@ export default class ProductHuntConnector extends ConnectorRuntime {
265
265
 
266
266
  let result: ProductHuntPostsResponse;
267
267
  try {
268
- result = await http.post<ProductHuntPostsResponse>(this.API_URL, {
269
- query: LIST_POSTS_QUERY,
270
- variables,
271
- });
268
+ // A GraphQL query sent as POST: safe to repeat on a 5xx.
269
+ result = await http.post<ProductHuntPostsResponse>(
270
+ this.API_URL,
271
+ { query: LIST_POSTS_QUERY, variables },
272
+ { idempotent: true }
273
+ );
272
274
  } catch (error) {
273
275
  if (error instanceof HttpStatusError && error.status === 401) {
274
276
  // PH v2 API requires auth. Fail the run instead of returning an
@@ -352,9 +354,9 @@ export default class ProductHuntConnector extends ConnectorRuntime {
352
354
  last_sync_at: new Date().toISOString(),
353
355
  };
354
356
 
357
+ await ctx.commit(events, checkpoint as Record<string, unknown>);
355
358
  return {
356
- events,
357
- checkpoint: checkpoint as Record<string, unknown>,
359
+ status: 'complete',
358
360
  metadata: {
359
361
  items_found: events.length,
360
362
  },
@@ -362,9 +362,9 @@ export default class RedditConnector extends ConnectorRuntime {
362
362
  pagination_token: after ?? undefined,
363
363
  };
364
364
 
365
+ await ctx.commit(events, checkpoint as Record<string, unknown>);
365
366
  return {
366
- events,
367
- checkpoint: checkpoint as Record<string, unknown>,
367
+ status: 'complete',
368
368
  metadata: {
369
369
  items_found: events.length,
370
370
  },