@lobu/runtime-device 20.2.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +42 -36
- package/npm-shrinkwrap.json +270 -224
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
- package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/github.ts +15 -9
- package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +62 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +8 -6
- package/vendor/cli/dist/connectors/reddit.ts +2 -2
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +42 -51
- package/vendor/connector-worker/dist/daemon/start.js +5 -1
- package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
- package/vendor/connector-worker/dist/executor/isolate.js +45 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
- package/vendor/core/dist/contracts/worker/protocol.js +58 -9
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +25 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -457,7 +457,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
457
457
|
const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
|
|
458
458
|
|
|
459
459
|
const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
|
|
460
|
-
const events: EventEnvelope[] = [];
|
|
461
460
|
const durableChanges = ctx.feedKey === 'changes';
|
|
462
461
|
|
|
463
462
|
// A checkpoint minted under a different scope (or by an older version that
|
|
@@ -467,10 +466,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
467
466
|
const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
|
|
468
467
|
const resumable = checkpoint.scope === scope;
|
|
469
468
|
|
|
470
|
-
/** Traversal reached the last page:
|
|
471
|
-
const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
|
|
472
|
-
|
|
473
|
-
|
|
469
|
+
/** Traversal reached the last page: commit it with its cursor and stamp the run. */
|
|
470
|
+
const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
|
|
471
|
+
await ctx.commit(items, {
|
|
472
|
+
...this.completedCheckpoint(syncToken),
|
|
473
|
+
scope,
|
|
474
|
+
} as Record<string, unknown>);
|
|
475
|
+
return { status: 'complete' };
|
|
474
476
|
};
|
|
475
477
|
|
|
476
478
|
if (resumable && checkpoint.sync_token) {
|
|
@@ -517,15 +519,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
517
519
|
const baseParams = params.toString();
|
|
518
520
|
|
|
519
521
|
/**
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
522
|
+
* Mid-traversal: the next page token, with no sync token and no
|
|
523
|
+
* `last_sync_at`, so nothing downstream can read it as a completed window.
|
|
524
|
+
* A run that stops here resumes from `token`.
|
|
523
525
|
*/
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
checkpoint: { scope, pending: { params: baseParams, page_token: token } },
|
|
527
|
-
});
|
|
526
|
+
const parked = (token: string) =>
|
|
527
|
+
({ scope, pending: { params: baseParams, page_token: token } }) as Record<string, unknown>;
|
|
528
528
|
|
|
529
|
+
let collected = 0;
|
|
529
530
|
let pageToken = pending?.page_token;
|
|
530
531
|
const seenTokens = new Set<string>();
|
|
531
532
|
for (let page = 0; ; page++) {
|
|
@@ -549,6 +550,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
549
550
|
// A provider page is atomic: `max_results` caps how much a run starts,
|
|
550
551
|
// never how much of a fetched page is stored. Dropping a page's tail
|
|
551
552
|
// would lose those events for good once the cursor moved past them.
|
|
553
|
+
const events: EventEnvelope[] = [];
|
|
552
554
|
for (const calEvent of items) {
|
|
553
555
|
const envelope = durableChanges
|
|
554
556
|
? this.calendarEventToChangeEnvelope(calEvent)
|
|
@@ -570,17 +572,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
570
572
|
throw new Error('Google Calendar returned a repeated page token.');
|
|
571
573
|
}
|
|
572
574
|
pageToken = data.nextPageToken;
|
|
575
|
+
await ctx.commit(events, parked(pageToken));
|
|
576
|
+
collected += events.length;
|
|
573
577
|
|
|
574
578
|
// Stop once this run has collected its share, once the provider returns
|
|
575
579
|
// an empty page, or at the hard ceiling — the last of which is the only
|
|
576
580
|
// bound on a window of nothing but filtered-out items, where
|
|
577
|
-
// `
|
|
581
|
+
// `collected` never reaches the cap. The traversal continues next run.
|
|
578
582
|
if (
|
|
579
|
-
|
|
583
|
+
collected >= maxResults ||
|
|
580
584
|
items.length === 0 ||
|
|
581
585
|
page + 1 >= MAX_SYNC_PAGES
|
|
582
586
|
) {
|
|
583
|
-
return
|
|
587
|
+
return { status: 'more' };
|
|
584
588
|
}
|
|
585
589
|
}
|
|
586
590
|
}
|
|
@@ -985,29 +989,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
985
989
|
};
|
|
986
990
|
}
|
|
987
991
|
|
|
988
|
-
private
|
|
989
|
-
events: EventEnvelope[],
|
|
990
|
-
syncToken: string | undefined
|
|
991
|
-
): SyncResult {
|
|
992
|
-
// Sort events by occurred_at descending
|
|
993
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
994
|
-
|
|
992
|
+
private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
|
|
995
993
|
// Built fresh rather than spread over the previous checkpoint: the whole
|
|
996
994
|
// object replaces the stored one, so a run that recovered from a rejected
|
|
997
995
|
// token cannot leave that token behind. When the sync produced no new token
|
|
998
996
|
// the key is absent, and the next run correctly starts from a full sync.
|
|
999
|
-
|
|
997
|
+
return {
|
|
1000
998
|
...(syncToken ? { sync_token: syncToken } : {}),
|
|
1001
999
|
last_sync_at: new Date().toISOString(),
|
|
1002
1000
|
};
|
|
1003
|
-
|
|
1004
|
-
return {
|
|
1005
|
-
events,
|
|
1006
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1007
|
-
metadata: {
|
|
1008
|
-
items_found: events.length,
|
|
1009
|
-
},
|
|
1010
|
-
};
|
|
1011
1001
|
}
|
|
1012
1002
|
|
|
1013
1003
|
// Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
|
|
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
|
|
|
286
286
|
// Connector
|
|
287
287
|
// ---------------------------------------------------------------------------
|
|
288
288
|
|
|
289
|
+
type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Checkpoint for a bootstrap with list pages still unread.
|
|
293
|
+
*
|
|
294
|
+
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
295
|
+
* incremental, which is exactly the property that keeps the unread files
|
|
296
|
+
* reachable.
|
|
297
|
+
*/
|
|
298
|
+
function bootstrapCheckpoint(
|
|
299
|
+
pendingPageToken: string,
|
|
300
|
+
listPageToken: string,
|
|
301
|
+
startedAt: string
|
|
302
|
+
): DriveCheckpoint {
|
|
303
|
+
return {
|
|
304
|
+
pending_page_token: pendingPageToken,
|
|
305
|
+
list_page_token: listPageToken,
|
|
306
|
+
last_sync_at: startedAt,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
289
310
|
export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
290
311
|
Record<string, unknown>,
|
|
291
312
|
DriveConfig
|
|
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
493
514
|
|
|
494
515
|
// An unfinished bootstrap outranks everything: going incremental with pages
|
|
495
516
|
// still unread would strand every file the traversal never reached.
|
|
517
|
+
const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
|
|
518
|
+
ctx.commit(events, next as Record<string, unknown>);
|
|
519
|
+
|
|
496
520
|
if (checkpoint.list_page_token && checkpoint.pending_page_token) {
|
|
497
|
-
return this.fullSync(http, config, {
|
|
521
|
+
return this.fullSync(http, config, commit, {
|
|
498
522
|
startToken: checkpoint.pending_page_token,
|
|
499
523
|
listPageToken: checkpoint.list_page_token,
|
|
500
524
|
startedAt: checkpoint.last_sync_at,
|
|
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
506
530
|
http,
|
|
507
531
|
checkpoint.page_token,
|
|
508
532
|
config,
|
|
533
|
+
commit,
|
|
509
534
|
checkpoint.last_sync_at
|
|
510
535
|
);
|
|
511
|
-
if (result)
|
|
512
|
-
// A partial run must NOT advance `last_sync_at`. The view-churn guard
|
|
513
|
-
// skips content for files modified before that stamp, so stamping now
|
|
514
|
-
// would make the next run treat the unread backlog as already-stored
|
|
515
|
-
// and drop its content on the floor.
|
|
516
|
-
return this.buildResult(
|
|
517
|
-
result.events,
|
|
518
|
-
result.nextPageToken,
|
|
519
|
-
result.partial ? checkpoint.last_sync_at : undefined,
|
|
520
|
-
{ changesPending: result.partial }
|
|
521
|
-
);
|
|
522
|
-
}
|
|
536
|
+
if (result) return result;
|
|
523
537
|
// Token rejected (see isPageTokenRejection). Fall through to ONE full
|
|
524
538
|
// re-list, exactly as the Calendar connector does — never a retry loop.
|
|
525
539
|
}
|
|
526
540
|
|
|
527
|
-
return this.fullSync(http, config);
|
|
541
|
+
return this.fullSync(http, config, commit);
|
|
528
542
|
}
|
|
529
543
|
|
|
530
544
|
/**
|
|
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
539
553
|
private async fullSync(
|
|
540
554
|
http: HttpClient,
|
|
541
555
|
config: DriveConfig,
|
|
556
|
+
commit: DriveCommit,
|
|
542
557
|
resume?: { startToken: string; listPageToken: string; startedAt?: string }
|
|
543
558
|
): Promise<SyncResult> {
|
|
544
559
|
// A resumed bootstrap keeps the token minted when the traversal STARTED —
|
|
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
555
570
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
556
571
|
const includeContent = config.include_content !== false;
|
|
557
572
|
const q = buildListQuery(config);
|
|
558
|
-
|
|
573
|
+
let collected = 0;
|
|
559
574
|
|
|
560
575
|
// 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
|
|
561
576
|
// configurable max_results. Defensive bound against a self-referential
|
|
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
605
620
|
{ maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
|
|
606
621
|
);
|
|
607
622
|
|
|
623
|
+
// A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
|
|
624
|
+
// client-side against a change record, so the only way to keep honouring it
|
|
625
|
+
// is to keep listing. Withholding the token re-lists next run; promoting it
|
|
626
|
+
// would silently widen the feed to every changed file in the Drive.
|
|
627
|
+
const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
|
|
628
|
+
|
|
608
629
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
609
630
|
for await (const items of pages) {
|
|
631
|
+
const events: EventEnvelope[] = [];
|
|
610
632
|
for (const file of items) {
|
|
611
633
|
events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
|
|
612
634
|
}
|
|
613
|
-
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
635
|
+
collected += events.length;
|
|
636
|
+
// Only Drive withholding a nextPageToken proves the traversal exhausted,
|
|
637
|
+
// so that — not the reason a run stops — is what may promote the change
|
|
638
|
+
// token. Until then each page commits with the bootstrap parked on the
|
|
639
|
+
// next unread page: a page is consumed whole or not at all, so the list
|
|
640
|
+
// token always addresses the first file not yet stored.
|
|
641
|
+
if (!nextListPageToken) {
|
|
642
|
+
await commit(events, {
|
|
643
|
+
...(hasCustomQuery ? {} : { page_token: startToken }),
|
|
644
|
+
last_sync_at: bootstrapStartedAt,
|
|
645
|
+
});
|
|
646
|
+
return {
|
|
647
|
+
status: 'complete',
|
|
648
|
+
metadata: {
|
|
649
|
+
items_found: collected,
|
|
650
|
+
bootstrap_complete: true,
|
|
651
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
652
|
+
},
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
|
|
656
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
632
657
|
}
|
|
633
658
|
|
|
634
|
-
//
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
659
|
+
// Unread pages remain, and WHY this run stopped is irrelevant: the item
|
|
660
|
+
// cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
|
|
661
|
+
// the change feed will never replay. The bootstrap resumes next run.
|
|
662
|
+
return {
|
|
663
|
+
status: 'more',
|
|
664
|
+
metadata: {
|
|
665
|
+
items_found: collected,
|
|
666
|
+
bootstrap_complete: false,
|
|
667
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
668
|
+
},
|
|
669
|
+
};
|
|
645
670
|
}
|
|
646
671
|
|
|
647
672
|
/**
|
|
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
654
679
|
http: HttpClient,
|
|
655
680
|
pageToken: string,
|
|
656
681
|
config: DriveConfig,
|
|
682
|
+
commit: DriveCommit,
|
|
657
683
|
lastSyncAt?: string
|
|
658
|
-
): Promise<{
|
|
659
|
-
events: EventEnvelope[];
|
|
660
|
-
nextPageToken?: string;
|
|
661
|
-
partial: boolean;
|
|
662
|
-
} | null> {
|
|
684
|
+
): Promise<SyncResult | null> {
|
|
663
685
|
const includeContent = config.include_content !== false;
|
|
664
686
|
const includeTrashed = Boolean(config.include_trashed);
|
|
665
687
|
const folderId =
|
|
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
667
689
|
? config.folder_id.trim()
|
|
668
690
|
: undefined;
|
|
669
691
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
670
|
-
|
|
692
|
+
let collected = 0;
|
|
671
693
|
let nextPageToken: string | undefined;
|
|
672
694
|
// The cursor for the page AFTER the one just consumed. A run that stops
|
|
673
695
|
// early persists this instead of dropping back to a full re-list — a
|
|
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
718
740
|
// start of the next page, so stopping part-way through one would skip the
|
|
719
741
|
// changes still in it.
|
|
720
742
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
721
|
-
let stoppedEarly = false;
|
|
722
743
|
for await (const items of pages) {
|
|
744
|
+
if (rejected) return null;
|
|
745
|
+
const events: EventEnvelope[] = [];
|
|
723
746
|
for (const change of items) {
|
|
724
747
|
const envelope = await this.changeToEnvelope(
|
|
725
748
|
http,
|
|
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
731
754
|
);
|
|
732
755
|
if (envelope) events.push(envelope);
|
|
733
756
|
}
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
757
|
+
collected += events.length;
|
|
758
|
+
if (!resumeCursor) {
|
|
759
|
+
// The stream is caught up: the page that carried `newStartPageToken`
|
|
760
|
+
// commits it and stamps the run. Rebuilt rather than spread over the
|
|
761
|
+
// previous checkpoint so a run that recovered from a rejected token
|
|
762
|
+
// cannot leave that token behind; without a new token the key is
|
|
763
|
+
// absent and the next run correctly re-lists.
|
|
764
|
+
await commit(events, {
|
|
765
|
+
...(nextPageToken ? { page_token: nextPageToken } : {}),
|
|
766
|
+
last_sync_at: new Date().toISOString(),
|
|
767
|
+
});
|
|
768
|
+
return {
|
|
769
|
+
status: 'complete',
|
|
770
|
+
metadata: { items_found: collected, bootstrap_complete: true },
|
|
771
|
+
};
|
|
738
772
|
}
|
|
773
|
+
// Mid-stream: commit with the cursor of the next page, a resumable
|
|
774
|
+
// position in the change stream. `last_sync_at` does NOT advance — the
|
|
775
|
+
// view-churn guard skips content for files modified before that stamp,
|
|
776
|
+
// so stamping now would make the next run treat the unread backlog as
|
|
777
|
+
// already stored and drop its content on the floor.
|
|
778
|
+
await commit(events, {
|
|
779
|
+
page_token: resumeCursor,
|
|
780
|
+
...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
|
|
781
|
+
});
|
|
782
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
739
783
|
}
|
|
740
|
-
|
|
741
784
|
if (rejected) return null;
|
|
742
785
|
|
|
743
|
-
//
|
|
744
|
-
// stream still open
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
return { events, nextPageToken: resumeCursor, partial: true };
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
return { events, nextPageToken, partial: false };
|
|
786
|
+
// Stopped on the cap or the clock, or the paginator ran out of pages
|
|
787
|
+
// (MAX_PAGES) with the stream still open. The committed cursor resumes it.
|
|
788
|
+
return {
|
|
789
|
+
status: 'more',
|
|
790
|
+
metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
|
|
791
|
+
};
|
|
753
792
|
}
|
|
754
793
|
|
|
755
794
|
private async fetchStartPageToken(http: HttpClient): Promise<string> {
|
|
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
1219
1258
|
};
|
|
1220
1259
|
}
|
|
1221
1260
|
|
|
1222
|
-
/**
|
|
1223
|
-
* Checkpoint for a bootstrap that stopped on `max_results` with pages left.
|
|
1224
|
-
*
|
|
1225
|
-
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
1226
|
-
* incremental, which is exactly the property that keeps the unread files
|
|
1227
|
-
* reachable. `bootstrap_complete: false` is surfaced in metadata so an
|
|
1228
|
-
* operator can see the traversal is still catching up rather than guessing
|
|
1229
|
-
* from a suspiciously round item count.
|
|
1230
|
-
*/
|
|
1231
|
-
private buildBootstrapResult(
|
|
1232
|
-
events: EventEnvelope[],
|
|
1233
|
-
pendingPageToken: string,
|
|
1234
|
-
listPageToken: string,
|
|
1235
|
-
startedAt: string,
|
|
1236
|
-
searchIncomplete = false
|
|
1237
|
-
): SyncResult {
|
|
1238
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1239
|
-
|
|
1240
|
-
const checkpoint: DriveCheckpoint = {
|
|
1241
|
-
pending_page_token: pendingPageToken,
|
|
1242
|
-
list_page_token: listPageToken,
|
|
1243
|
-
last_sync_at: startedAt,
|
|
1244
|
-
};
|
|
1245
|
-
|
|
1246
|
-
return {
|
|
1247
|
-
events,
|
|
1248
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
1249
|
-
metadata: {
|
|
1250
|
-
items_found: events.length,
|
|
1251
|
-
bootstrap_complete: false,
|
|
1252
|
-
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
1253
|
-
},
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
private buildResult(
|
|
1258
|
-
events: EventEnvelope[],
|
|
1259
|
-
pageToken: string | undefined,
|
|
1260
|
-
/** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
|
|
1261
|
-
keepLastSyncAt: string | undefined,
|
|
1262
|
-
flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
|
|
1263
|
-
): SyncResult {
|
|
1264
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1265
|
-
|
|
1266
|
-
// Rebuilt rather than spread over the previous checkpoint so a run that
|
|
1267
|
-
// recovered from a rejected token cannot leave that token behind. Without a
|
|
1268
|
-
// new token the key is absent and the next run correctly re-lists.
|
|
1269
|
-
const newCheckpoint: DriveCheckpoint = {
|
|
1270
|
-
...(pageToken ? { page_token: pageToken } : {}),
|
|
1271
|
-
last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
|
|
1272
|
-
};
|
|
1273
|
-
|
|
1274
|
-
return {
|
|
1275
|
-
events,
|
|
1276
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1277
|
-
metadata: {
|
|
1278
|
-
items_found: events.length,
|
|
1279
|
-
bootstrap_complete: true,
|
|
1280
|
-
...(flags.changesPending ? { changes_pending: true } : {}),
|
|
1281
|
-
...(flags.searchIncomplete ? { search_incomplete: true } : {}),
|
|
1282
|
-
},
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
1261
|
/** Overridable so a test can drive the time budget without sleeping. */
|
|
1287
1262
|
protected now(): number {
|
|
1288
1263
|
return Date.now();
|
|
@@ -100,6 +100,8 @@ interface GmailCheckpoint {
|
|
|
100
100
|
/** Serialized search + person filter the two fields below were produced under. */
|
|
101
101
|
scope?: string;
|
|
102
102
|
last_sync_at?: string;
|
|
103
|
+
/** Undecodable threads, keyed by thread ID; retained until a later successful read. */
|
|
104
|
+
skipped_threads?: Record<string, { message_id: string; error: string }>;
|
|
103
105
|
/** Set while a window is only part-walked: the run hit `max_results`, or a page came back empty with a cursor. */
|
|
104
106
|
pending?: {
|
|
105
107
|
query: string;
|
|
@@ -108,6 +110,12 @@ interface GmailCheckpoint {
|
|
|
108
110
|
};
|
|
109
111
|
}
|
|
110
112
|
|
|
113
|
+
class GmailBodyDecodeError extends Error {
|
|
114
|
+
constructor(readonly messageId: string, encoding: string, cause: unknown) {
|
|
115
|
+
super(`Gmail body decode (${encoding}): ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
111
119
|
interface GmailConfig {
|
|
112
120
|
/**
|
|
113
121
|
* Gmail search scope shared by sync and direct source reads. Takes precedence
|
|
@@ -449,7 +457,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
449
457
|
// sync
|
|
450
458
|
// -------------------------------------------------------------------------
|
|
451
459
|
|
|
452
|
-
private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult
|
|
460
|
+
private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult> {
|
|
453
461
|
const syncStartedAt = new Date();
|
|
454
462
|
const token = ctx.credentials?.accessToken;
|
|
455
463
|
if (!token) {
|
|
@@ -485,7 +493,23 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
485
493
|
// first page. `origin_id` makes that replay a supersede, not a duplicate.
|
|
486
494
|
let staleCursorRecoverable = pageToken !== undefined;
|
|
487
495
|
const http = this.createClient(token);
|
|
488
|
-
const
|
|
496
|
+
const skippedThreads = { ...checkpoint.skipped_threads };
|
|
497
|
+
let itemsFound = 0;
|
|
498
|
+
let itemsSkipped = 0;
|
|
499
|
+
// A part-walked window keeps the previous `last_sync_at`: it may not advance
|
|
500
|
+
// until the whole window has been walked, or the unread tail is skipped.
|
|
501
|
+
const checkpointAt = (cursor: string | undefined): GmailCheckpoint => {
|
|
502
|
+
const next: GmailCheckpoint = cursor
|
|
503
|
+
? {
|
|
504
|
+
schema_version: 3,
|
|
505
|
+
scope: scopeKey,
|
|
506
|
+
...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
507
|
+
pending: { query, started_at: windowStart, page_token: cursor },
|
|
508
|
+
}
|
|
509
|
+
: { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
|
|
510
|
+
if (Object.keys(skippedThreads).length) next.skipped_threads = { ...skippedThreads };
|
|
511
|
+
return next;
|
|
512
|
+
};
|
|
489
513
|
// Bounds the threads FETCHED per run (each costs at least one API call),
|
|
490
514
|
// independent of how many survive the person filter — a narrow feed must not
|
|
491
515
|
// scan the whole window just because most threads are rejected. It also sizes
|
|
@@ -513,6 +537,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
513
537
|
staleCursorRecoverable = false;
|
|
514
538
|
const listData = (await listResponse.json()) as GmailThreadListResponse;
|
|
515
539
|
const threads = listData.threads ?? [];
|
|
540
|
+
const events: EventEnvelope[] = [];
|
|
516
541
|
for (const threadStub of threads) {
|
|
517
542
|
inspected++;
|
|
518
543
|
try {
|
|
@@ -562,6 +587,13 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
562
587
|
...(attribution.fromName ? { from_name: attribution.fromName } : {}),
|
|
563
588
|
},
|
|
564
589
|
});
|
|
590
|
+
delete skippedThreads[threadStub.id];
|
|
591
|
+
} catch (error) {
|
|
592
|
+
// Only decoding failures are permanent for these bytes. Fetch and
|
|
593
|
+
// conversion failures must still retry the window without advancing.
|
|
594
|
+
if (!(error instanceof GmailBodyDecodeError)) throw error;
|
|
595
|
+
skippedThreads[threadStub.id] = { message_id: error.messageId, error: error.message };
|
|
596
|
+
itemsSkipped++;
|
|
565
597
|
} finally {
|
|
566
598
|
await sleep(this.RATE_LIMIT_MS);
|
|
567
599
|
}
|
|
@@ -570,31 +602,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
570
602
|
throw new Error('Gmail returned a repeated page token');
|
|
571
603
|
}
|
|
572
604
|
pageToken = listData.nextPageToken;
|
|
573
|
-
//
|
|
574
|
-
//
|
|
575
|
-
|
|
576
|
-
|
|
605
|
+
// The page commits with the cursor of the next one, so a run cut short
|
|
606
|
+
// resumes where it stopped; the last page commits the finished window.
|
|
607
|
+
await ctx.commit(events, checkpointAt(pageToken));
|
|
608
|
+
itemsFound += events.length;
|
|
609
|
+
const metadata = { items_found: itemsFound, items_skipped: itemsSkipped };
|
|
610
|
+
if (!pageToken) return { status: 'complete', metadata };
|
|
611
|
+
// An empty page with a cursor is still incomplete. Its cursor is saved
|
|
612
|
+
// and the next run continues immediately rather than reporting the
|
|
613
|
+
// fixed window as caught up.
|
|
614
|
+
if (threads.length === 0) return { status: 'more', metadata };
|
|
615
|
+
if (inspected >= maxResults) return { status: 'more', metadata };
|
|
577
616
|
}
|
|
578
|
-
|
|
579
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
580
|
-
// A part-walked window keeps the previous `last_sync_at`: it may not advance
|
|
581
|
-
// until the whole window has been walked, or the unread tail is skipped.
|
|
582
|
-
const newCheckpoint: GmailCheckpoint = pageToken
|
|
583
|
-
? {
|
|
584
|
-
schema_version: 3,
|
|
585
|
-
scope: scopeKey,
|
|
586
|
-
...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
587
|
-
pending: { query, started_at: windowStart, page_token: pageToken },
|
|
588
|
-
}
|
|
589
|
-
: { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
|
|
590
|
-
|
|
591
|
-
return {
|
|
592
|
-
events,
|
|
593
|
-
checkpoint: newCheckpoint,
|
|
594
|
-
metadata: {
|
|
595
|
-
items_found: events.length,
|
|
596
|
-
},
|
|
597
|
-
};
|
|
598
617
|
}
|
|
599
618
|
|
|
600
619
|
// -------------------------------------------------------------------------
|
|
@@ -1356,7 +1375,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1356
1375
|
if (mimeType !== 'text/plain' && mimeType !== 'text/html') return '';
|
|
1357
1376
|
let text = '';
|
|
1358
1377
|
if (payload.body?.data) {
|
|
1359
|
-
text = this.base64UrlDecode(payload.body.data, payload);
|
|
1378
|
+
text = this.base64UrlDecode(payload.body.data, payload, messageId);
|
|
1360
1379
|
} else if (payload.body?.attachmentId) {
|
|
1361
1380
|
// Gmail may externalize the body itself, even under format=full. A
|
|
1362
1381
|
// failed fetch must not silently replace that body with its alternative.
|
|
@@ -1364,7 +1383,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1364
1383
|
if (!response.ok) throw new Error(`Gmail message body error (${response.status}): ${await response.text()}`);
|
|
1365
1384
|
const body = (await response.json()) as { data?: string };
|
|
1366
1385
|
if (typeof body.data !== 'string') throw new Error('Gmail returned a message body without data');
|
|
1367
|
-
text = this.base64UrlDecode(body.data, payload);
|
|
1386
|
+
text = this.base64UrlDecode(body.data, payload, messageId);
|
|
1368
1387
|
}
|
|
1369
1388
|
if (!text && payload.body?.size) throw new Error('Gmail returned a nonempty message body without data');
|
|
1370
1389
|
// Convert each HTML leaf after charset decoding, preserving plain parts,
|
|
@@ -1374,14 +1393,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1374
1393
|
return text.trim() ? text : '';
|
|
1375
1394
|
}
|
|
1376
1395
|
|
|
1377
|
-
private base64UrlDecode(data: string, payload: GmailMessagePayload): string {
|
|
1396
|
+
private base64UrlDecode(data: string, payload: GmailMessagePayload, messageId: string): string {
|
|
1378
1397
|
const padded = data.replace(/-/g, '+').replace(/_/g, '/');
|
|
1379
1398
|
const contentType = payload.headers?.find((header) => header.name.toLowerCase() === 'content-type')?.value;
|
|
1380
1399
|
const charset = contentType?.match(/;\s*charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\s]+))/i);
|
|
1381
1400
|
const encoding = (charset?.[1] ?? charset?.[2] ?? charset?.[3] ?? 'utf-8').trim();
|
|
1382
1401
|
// MIME bodies contain bytes in the part's declared charset. Invalid bytes
|
|
1383
1402
|
// or unsupported labels must fail the read instead of persisting corruption.
|
|
1384
|
-
|
|
1403
|
+
try {
|
|
1404
|
+
return new TextDecoder(encoding, { fatal: true }).decode(Buffer.from(padded, 'base64'));
|
|
1405
|
+
} catch (cause) {
|
|
1406
|
+
throw new GmailBodyDecodeError(messageId, encoding, cause);
|
|
1407
|
+
}
|
|
1385
1408
|
}
|
|
1386
1409
|
|
|
1387
1410
|
private base64UrlEncode(str: string): string {
|
|
@@ -376,11 +376,8 @@ export default class HackerNewsConnector extends ConnectorRuntime {
|
|
|
376
376
|
await this.enrichStoriesWithExternalContent(events, deadline);
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
checkpoint: { last_sync_at: new Date().toISOString() },
|
|
382
|
-
metadata: { items_found: events.length },
|
|
383
|
-
};
|
|
379
|
+
await ctx.commit(events, { last_sync_at: new Date().toISOString() });
|
|
380
|
+
return { status: 'complete', metadata: { items_found: events.length } };
|
|
384
381
|
}
|
|
385
382
|
|
|
386
383
|
// -------------------------------------------------------------------------
|