@lobu/runtime-device 20.2.0 → 21.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +44 -38
- package/npm-shrinkwrap.json +277 -231
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -79
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +41 -29
- package/vendor/cli/dist/commands/_lib/apply/client.js +33 -47
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -13
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -29
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +211 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/commands/memory/_lib/schema.d.ts +0 -1
- package/vendor/cli/dist/commands/memory/_lib/schema.js +0 -9
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +0 -2
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/github.ts +274 -193
- package/vendor/cli/dist/connectors/google_calendar.ts +54 -160
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +71 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +11 -7
- package/vendor/cli/dist/connectors/reddit.ts +36 -6
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/providers.json +0 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -15
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/client.d.ts +4 -1
- package/vendor/connector-worker/dist/daemon/client.js +3 -0
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +46 -51
- package/vendor/connector-worker/dist/daemon/start.js +6 -2
- package/vendor/connector-worker/dist/executor/interface.d.ts +19 -9
- package/vendor/connector-worker/dist/executor/isolate.js +85 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-automations.d.ts +0 -10
- package/vendor/core/dist/contracts/tools/manage-automations.js +0 -33
- package/vendor/core/dist/contracts/tools/manage-classifiers.d.ts +4 -34
- package/vendor/core/dist/contracts/tools/manage-classifiers.js +13 -44
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +0 -9
- package/vendor/core/dist/contracts/tools/manage-entity.js +0 -6
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-attach.d.ts +20 -0
- package/vendor/core/dist/contracts/tools/view-attach.js +45 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +90 -8
- package/vendor/core/dist/contracts/worker/protocol.js +70 -10
- package/vendor/core/dist/index.d.ts +1 -1
- package/vendor/core/dist/provider-config-types.d.ts +0 -9
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/sdk-compat.d.ts +1 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +30 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -12,7 +12,6 @@ import {
|
|
|
12
12
|
createHttpClient,
|
|
13
13
|
type EventEnvelope,
|
|
14
14
|
type HttpClient,
|
|
15
|
-
paginateByCursor,
|
|
16
15
|
type FeedReadContext,
|
|
17
16
|
type FeedReadResult,
|
|
18
17
|
type RuntimeConnectorDefinition,
|
|
@@ -82,7 +81,7 @@ interface CalendarCheckpoint {
|
|
|
82
81
|
*/
|
|
83
82
|
scope?: string;
|
|
84
83
|
/**
|
|
85
|
-
* Unfinished
|
|
84
|
+
* Unfinished traversal: the `events.list` query as first issued (so the
|
|
86
85
|
* `timeMin`/`timeMax` window stays fixed across runs, which the page token
|
|
87
86
|
* requires) plus the token for the page still to fetch. Absent once the
|
|
88
87
|
* traversal has reached the last page.
|
|
@@ -457,7 +456,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
457
456
|
const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
|
|
458
457
|
|
|
459
458
|
const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
|
|
460
|
-
const events: EventEnvelope[] = [];
|
|
461
459
|
const durableChanges = ctx.feedKey === 'changes';
|
|
462
460
|
|
|
463
461
|
// A checkpoint minted under a different scope (or by an older version that
|
|
@@ -467,28 +465,18 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
467
465
|
const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
|
|
468
466
|
const resumable = checkpoint.scope === scope;
|
|
469
467
|
|
|
470
|
-
/** Traversal reached the last page:
|
|
471
|
-
const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
|
|
472
|
-
|
|
473
|
-
|
|
468
|
+
/** Traversal reached the last page: commit it with its cursor and stamp the run. */
|
|
469
|
+
const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
|
|
470
|
+
await ctx.commit(items, {
|
|
471
|
+
...this.completedCheckpoint(syncToken),
|
|
472
|
+
scope,
|
|
473
|
+
} as Record<string, unknown>);
|
|
474
|
+
return { status: 'complete' };
|
|
474
475
|
};
|
|
475
476
|
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
calendarId,
|
|
480
|
-
checkpoint.sync_token,
|
|
481
|
-
durableChanges
|
|
482
|
-
);
|
|
483
|
-
if (result) return finish(result.events, result.nextSyncToken);
|
|
484
|
-
// The stored token was rejected (see isSyncTokenRejection). Fall through
|
|
485
|
-
// to one complete bootstrap under the current grant — no retry loop. It
|
|
486
|
-
// either succeeds and overwrites the checkpoint with a token minted under
|
|
487
|
-
// that grant, or throws and the error reaches the run record.
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
// Bootstrap: walk the configured window, resuming a parked page if one is
|
|
491
|
-
// in scope, until the last page hands over a durable cursor.
|
|
477
|
+
const syncToken = resumable ? checkpoint.sync_token : undefined;
|
|
478
|
+
// Bootstrap and incremental traversal share the same page commit boundary.
|
|
479
|
+
// An incremental page keeps its original sync token until the final page.
|
|
492
480
|
const timeMin = new Date();
|
|
493
481
|
timeMin.setDate(timeMin.getDate() - lookbackDays);
|
|
494
482
|
const timeMax = new Date();
|
|
@@ -500,32 +488,41 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
500
488
|
// run: a page token is only valid against the window that minted it.
|
|
501
489
|
const params = pending
|
|
502
490
|
? new URLSearchParams(pending.params)
|
|
503
|
-
:
|
|
491
|
+
: syncToken
|
|
504
492
|
? new URLSearchParams({
|
|
505
493
|
maxResults: '250',
|
|
506
|
-
|
|
507
|
-
showDeleted: 'true',
|
|
508
|
-
timeMin: timeMin.toISOString(),
|
|
494
|
+
syncToken,
|
|
495
|
+
...(durableChanges ? { singleEvents: 'true', showDeleted: 'true' } : {}),
|
|
509
496
|
})
|
|
510
|
-
:
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
497
|
+
: durableChanges
|
|
498
|
+
? new URLSearchParams({
|
|
499
|
+
maxResults: '250',
|
|
500
|
+
singleEvents: 'true',
|
|
501
|
+
showDeleted: 'true',
|
|
502
|
+
timeMin: timeMin.toISOString(),
|
|
503
|
+
})
|
|
504
|
+
: new URLSearchParams({
|
|
505
|
+
maxResults: '250',
|
|
506
|
+
orderBy: 'startTime',
|
|
507
|
+
singleEvents: 'true',
|
|
508
|
+
timeMin: timeMin.toISOString(),
|
|
509
|
+
timeMax: timeMax.toISOString(),
|
|
510
|
+
});
|
|
517
511
|
const baseParams = params.toString();
|
|
518
512
|
|
|
519
513
|
/**
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
514
|
+
* Mid-traversal: preserve the old sync token and completed watermark,
|
|
515
|
+
* alongside the next page and the exact query that minted it.
|
|
516
|
+
* A run that stops here resumes from `token`.
|
|
523
517
|
*/
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
518
|
+
const parked = (token: string) =>
|
|
519
|
+
({
|
|
520
|
+
scope,
|
|
521
|
+
...(syncToken ? { sync_token: syncToken, last_sync_at: checkpoint.last_sync_at } : {}),
|
|
522
|
+
pending: { params: baseParams, page_token: token },
|
|
523
|
+
}) as Record<string, unknown>;
|
|
524
|
+
|
|
525
|
+
let collected = 0;
|
|
529
526
|
let pageToken = pending?.page_token;
|
|
530
527
|
const seenTokens = new Set<string>();
|
|
531
528
|
for (let page = 0; ; page++) {
|
|
@@ -537,11 +534,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
537
534
|
const url = `${this.BASE_URL}/calendars/${encodeURIComponent(calendarId)}/events?${params.toString()}`;
|
|
538
535
|
const response = await http.raw(url);
|
|
539
536
|
if (!response.ok) {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
537
|
+
const body = await response.text();
|
|
538
|
+
if (syncToken && isSyncTokenRejection(response.status, body)) {
|
|
539
|
+
// Discard both tokens together. Replayed items keep their origin IDs;
|
|
540
|
+
// a failed bootstrap remains retryable without treating it as complete.
|
|
541
|
+
return this.syncFeed({ ...ctx, checkpoint: { scope } });
|
|
542
|
+
}
|
|
543
|
+
throw new Error(`Calendar events.list error (${response.status}): ${body}`);
|
|
545
544
|
}
|
|
546
545
|
|
|
547
546
|
const data = (await response.json()) as CalendarEventListResponse;
|
|
@@ -549,6 +548,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
549
548
|
// A provider page is atomic: `max_results` caps how much a run starts,
|
|
550
549
|
// never how much of a fetched page is stored. Dropping a page's tail
|
|
551
550
|
// would lose those events for good once the cursor moved past them.
|
|
551
|
+
const events: EventEnvelope[] = [];
|
|
552
552
|
for (const calEvent of items) {
|
|
553
553
|
const envelope = durableChanges
|
|
554
554
|
? this.calendarEventToChangeEnvelope(calEvent)
|
|
@@ -570,17 +570,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
570
570
|
throw new Error('Google Calendar returned a repeated page token.');
|
|
571
571
|
}
|
|
572
572
|
pageToken = data.nextPageToken;
|
|
573
|
+
await ctx.commit(events, parked(pageToken));
|
|
574
|
+
collected += events.length;
|
|
573
575
|
|
|
574
576
|
// Stop once this run has collected its share, once the provider returns
|
|
575
577
|
// an empty page, or at the hard ceiling — the last of which is the only
|
|
576
578
|
// bound on a window of nothing but filtered-out items, where
|
|
577
|
-
// `
|
|
579
|
+
// `collected` never reaches the cap. The traversal continues next run.
|
|
578
580
|
if (
|
|
579
|
-
|
|
581
|
+
(!syncToken && collected >= maxResults) ||
|
|
580
582
|
items.length === 0 ||
|
|
581
583
|
page + 1 >= MAX_SYNC_PAGES
|
|
582
584
|
) {
|
|
583
|
-
return
|
|
585
|
+
return { status: 'more' };
|
|
584
586
|
}
|
|
585
587
|
}
|
|
586
588
|
}
|
|
@@ -621,100 +623,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
621
623
|
}
|
|
622
624
|
}
|
|
623
625
|
|
|
624
|
-
// -------------------------------------------------------------------------
|
|
625
|
-
// Incremental sync
|
|
626
|
-
// -------------------------------------------------------------------------
|
|
627
|
-
|
|
628
|
-
private async syncWithToken(
|
|
629
|
-
http: HttpClient,
|
|
630
|
-
calendarId: string,
|
|
631
|
-
syncToken: string,
|
|
632
|
-
durableChanges: boolean
|
|
633
|
-
): Promise<{ events: EventEnvelope[]; nextSyncToken?: string } | null> {
|
|
634
|
-
const events: EventEnvelope[] = [];
|
|
635
|
-
let nextSyncToken: string | undefined;
|
|
636
|
-
let unreadPage: string | undefined;
|
|
637
|
-
const seenTokens = new Set<string>();
|
|
638
|
-
// The stored syncToken was rejected (410 expired, or 403 because it was
|
|
639
|
-
// minted under a superseded grant) — abort and let the caller drop it and
|
|
640
|
-
// fall through to a full sync. Signalled out of the generator via this flag.
|
|
641
|
-
let syncTokenRejected = false;
|
|
642
|
-
|
|
643
|
-
const pages = paginateByCursor<CalendarEvent, string>(
|
|
644
|
-
async (pageToken) => {
|
|
645
|
-
if (pageToken) {
|
|
646
|
-
if (seenTokens.has(pageToken)) {
|
|
647
|
-
throw new Error('Google Calendar returned a repeated page token.');
|
|
648
|
-
}
|
|
649
|
-
seenTokens.add(pageToken);
|
|
650
|
-
}
|
|
651
|
-
// Consume every incremental change before advancing the durable token,
|
|
652
|
-
// so the page size is fixed rather than derived from `max_results`,
|
|
653
|
-
// which limits only the historical bootstrap.
|
|
654
|
-
const params = durableChanges
|
|
655
|
-
? new URLSearchParams({
|
|
656
|
-
maxResults: '250',
|
|
657
|
-
syncToken,
|
|
658
|
-
singleEvents: 'true',
|
|
659
|
-
showDeleted: 'true',
|
|
660
|
-
})
|
|
661
|
-
: new URLSearchParams({
|
|
662
|
-
maxResults: '250',
|
|
663
|
-
syncToken,
|
|
664
|
-
});
|
|
665
|
-
if (pageToken) {
|
|
666
|
-
params.set('pageToken', pageToken);
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
const url = `${this.BASE_URL}/calendars/${encodeURIComponent(calendarId)}/events?${params.toString()}`;
|
|
670
|
-
const response = await http.raw(url);
|
|
671
|
-
|
|
672
|
-
if (!response.ok) {
|
|
673
|
-
const body = await response.text();
|
|
674
|
-
if (isSyncTokenRejection(response.status, body)) {
|
|
675
|
-
syncTokenRejected = true;
|
|
676
|
-
return { items: [], nextCursor: undefined };
|
|
677
|
-
}
|
|
678
|
-
throw new Error(`Calendar events.list error (${response.status}): ${body}`);
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
const data = (await response.json()) as CalendarEventListResponse;
|
|
682
|
-
// Capture the trailing nextSyncToken; generator pages until exhausted.
|
|
683
|
-
nextSyncToken = data.nextSyncToken;
|
|
684
|
-
unreadPage = data.nextPageToken;
|
|
685
|
-
return { items: data.items ?? [], nextCursor: data.nextPageToken };
|
|
686
|
-
},
|
|
687
|
-
{ maxPages: MAX_SYNC_PAGES }
|
|
688
|
-
);
|
|
689
|
-
|
|
690
|
-
for await (const items of pages) {
|
|
691
|
-
for (const calEvent of items) {
|
|
692
|
-
const envelope = durableChanges
|
|
693
|
-
? this.calendarEventToChangeEnvelope(calEvent)
|
|
694
|
-
: this.calendarEventToEnvelope(calEvent);
|
|
695
|
-
if (envelope) events.push(envelope);
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
if (syncTokenRejected) return null;
|
|
700
|
-
// The generator stopped at MAX_SYNC_PAGES with a page still unread. Unlike
|
|
701
|
-
// the bootstrap there is nothing to park — an incremental cursor advances
|
|
702
|
-
// only on the last page — so fail the run and keep the stored token, rather
|
|
703
|
-
// than returning a silently partial batch.
|
|
704
|
-
if (unreadPage) {
|
|
705
|
-
throw new Error(
|
|
706
|
-
'Google Calendar incremental traversal exceeded its page bound before completing.'
|
|
707
|
-
);
|
|
708
|
-
}
|
|
709
|
-
if (durableChanges && !nextSyncToken) {
|
|
710
|
-
throw new Error(
|
|
711
|
-
'Google Calendar incremental changes traversal completed without a durable sync token.'
|
|
712
|
-
);
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
return { events, nextSyncToken };
|
|
716
|
-
}
|
|
717
|
-
|
|
718
626
|
// -------------------------------------------------------------------------
|
|
719
627
|
// Actions
|
|
720
628
|
// -------------------------------------------------------------------------
|
|
@@ -985,29 +893,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
985
893
|
};
|
|
986
894
|
}
|
|
987
895
|
|
|
988
|
-
private
|
|
989
|
-
events: EventEnvelope[],
|
|
990
|
-
syncToken: string | undefined
|
|
991
|
-
): SyncResult {
|
|
992
|
-
// Sort events by occurred_at descending
|
|
993
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
994
|
-
|
|
896
|
+
private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
|
|
995
897
|
// Built fresh rather than spread over the previous checkpoint: the whole
|
|
996
898
|
// object replaces the stored one, so a run that recovered from a rejected
|
|
997
899
|
// token cannot leave that token behind. When the sync produced no new token
|
|
998
900
|
// the key is absent, and the next run correctly starts from a full sync.
|
|
999
|
-
|
|
901
|
+
return {
|
|
1000
902
|
...(syncToken ? { sync_token: syncToken } : {}),
|
|
1001
903
|
last_sync_at: new Date().toISOString(),
|
|
1002
904
|
};
|
|
1003
|
-
|
|
1004
|
-
return {
|
|
1005
|
-
events,
|
|
1006
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1007
|
-
metadata: {
|
|
1008
|
-
items_found: events.length,
|
|
1009
|
-
},
|
|
1010
|
-
};
|
|
1011
905
|
}
|
|
1012
906
|
|
|
1013
907
|
// Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
|
|
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
|
|
|
286
286
|
// Connector
|
|
287
287
|
// ---------------------------------------------------------------------------
|
|
288
288
|
|
|
289
|
+
type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Checkpoint for a bootstrap with list pages still unread.
|
|
293
|
+
*
|
|
294
|
+
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
295
|
+
* incremental, which is exactly the property that keeps the unread files
|
|
296
|
+
* reachable.
|
|
297
|
+
*/
|
|
298
|
+
function bootstrapCheckpoint(
|
|
299
|
+
pendingPageToken: string,
|
|
300
|
+
listPageToken: string,
|
|
301
|
+
startedAt: string
|
|
302
|
+
): DriveCheckpoint {
|
|
303
|
+
return {
|
|
304
|
+
pending_page_token: pendingPageToken,
|
|
305
|
+
list_page_token: listPageToken,
|
|
306
|
+
last_sync_at: startedAt,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
289
310
|
export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
290
311
|
Record<string, unknown>,
|
|
291
312
|
DriveConfig
|
|
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
493
514
|
|
|
494
515
|
// An unfinished bootstrap outranks everything: going incremental with pages
|
|
495
516
|
// still unread would strand every file the traversal never reached.
|
|
517
|
+
const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
|
|
518
|
+
ctx.commit(events, next as Record<string, unknown>);
|
|
519
|
+
|
|
496
520
|
if (checkpoint.list_page_token && checkpoint.pending_page_token) {
|
|
497
|
-
return this.fullSync(http, config, {
|
|
521
|
+
return this.fullSync(http, config, commit, {
|
|
498
522
|
startToken: checkpoint.pending_page_token,
|
|
499
523
|
listPageToken: checkpoint.list_page_token,
|
|
500
524
|
startedAt: checkpoint.last_sync_at,
|
|
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
506
530
|
http,
|
|
507
531
|
checkpoint.page_token,
|
|
508
532
|
config,
|
|
533
|
+
commit,
|
|
509
534
|
checkpoint.last_sync_at
|
|
510
535
|
);
|
|
511
|
-
if (result)
|
|
512
|
-
// A partial run must NOT advance `last_sync_at`. The view-churn guard
|
|
513
|
-
// skips content for files modified before that stamp, so stamping now
|
|
514
|
-
// would make the next run treat the unread backlog as already-stored
|
|
515
|
-
// and drop its content on the floor.
|
|
516
|
-
return this.buildResult(
|
|
517
|
-
result.events,
|
|
518
|
-
result.nextPageToken,
|
|
519
|
-
result.partial ? checkpoint.last_sync_at : undefined,
|
|
520
|
-
{ changesPending: result.partial }
|
|
521
|
-
);
|
|
522
|
-
}
|
|
536
|
+
if (result) return result;
|
|
523
537
|
// Token rejected (see isPageTokenRejection). Fall through to ONE full
|
|
524
538
|
// re-list, exactly as the Calendar connector does — never a retry loop.
|
|
525
539
|
}
|
|
526
540
|
|
|
527
|
-
return this.fullSync(http, config);
|
|
541
|
+
return this.fullSync(http, config, commit);
|
|
528
542
|
}
|
|
529
543
|
|
|
530
544
|
/**
|
|
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
539
553
|
private async fullSync(
|
|
540
554
|
http: HttpClient,
|
|
541
555
|
config: DriveConfig,
|
|
556
|
+
commit: DriveCommit,
|
|
542
557
|
resume?: { startToken: string; listPageToken: string; startedAt?: string }
|
|
543
558
|
): Promise<SyncResult> {
|
|
544
559
|
// A resumed bootstrap keeps the token minted when the traversal STARTED —
|
|
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
555
570
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
556
571
|
const includeContent = config.include_content !== false;
|
|
557
572
|
const q = buildListQuery(config);
|
|
558
|
-
|
|
573
|
+
let collected = 0;
|
|
559
574
|
|
|
560
575
|
// 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
|
|
561
576
|
// configurable max_results. Defensive bound against a self-referential
|
|
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
605
620
|
{ maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
|
|
606
621
|
);
|
|
607
622
|
|
|
623
|
+
// A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
|
|
624
|
+
// client-side against a change record, so the only way to keep honouring it
|
|
625
|
+
// is to keep listing. Withholding the token re-lists next run; promoting it
|
|
626
|
+
// would silently widen the feed to every changed file in the Drive.
|
|
627
|
+
const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
|
|
628
|
+
|
|
608
629
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
609
630
|
for await (const items of pages) {
|
|
631
|
+
const events: EventEnvelope[] = [];
|
|
610
632
|
for (const file of items) {
|
|
611
633
|
events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
|
|
612
634
|
}
|
|
613
|
-
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
635
|
+
collected += events.length;
|
|
636
|
+
// Only Drive withholding a nextPageToken proves the traversal exhausted,
|
|
637
|
+
// so that — not the reason a run stops — is what may promote the change
|
|
638
|
+
// token. Until then each page commits with the bootstrap parked on the
|
|
639
|
+
// next unread page: a page is consumed whole or not at all, so the list
|
|
640
|
+
// token always addresses the first file not yet stored.
|
|
641
|
+
if (!nextListPageToken) {
|
|
642
|
+
await commit(events, {
|
|
643
|
+
...(hasCustomQuery ? {} : { page_token: startToken }),
|
|
644
|
+
last_sync_at: bootstrapStartedAt,
|
|
645
|
+
});
|
|
646
|
+
return {
|
|
647
|
+
status: 'complete',
|
|
648
|
+
metadata: {
|
|
649
|
+
items_found: collected,
|
|
650
|
+
bootstrap_complete: true,
|
|
651
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
652
|
+
},
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
|
|
656
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
632
657
|
}
|
|
633
658
|
|
|
634
|
-
//
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
659
|
+
// Unread pages remain, and WHY this run stopped is irrelevant: the item
|
|
660
|
+
// cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
|
|
661
|
+
// the change feed will never replay. The bootstrap resumes next run.
|
|
662
|
+
return {
|
|
663
|
+
status: 'more',
|
|
664
|
+
metadata: {
|
|
665
|
+
items_found: collected,
|
|
666
|
+
bootstrap_complete: false,
|
|
667
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
668
|
+
},
|
|
669
|
+
};
|
|
645
670
|
}
|
|
646
671
|
|
|
647
672
|
/**
|
|
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
654
679
|
http: HttpClient,
|
|
655
680
|
pageToken: string,
|
|
656
681
|
config: DriveConfig,
|
|
682
|
+
commit: DriveCommit,
|
|
657
683
|
lastSyncAt?: string
|
|
658
|
-
): Promise<{
|
|
659
|
-
events: EventEnvelope[];
|
|
660
|
-
nextPageToken?: string;
|
|
661
|
-
partial: boolean;
|
|
662
|
-
} | null> {
|
|
684
|
+
): Promise<SyncResult | null> {
|
|
663
685
|
const includeContent = config.include_content !== false;
|
|
664
686
|
const includeTrashed = Boolean(config.include_trashed);
|
|
665
687
|
const folderId =
|
|
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
667
689
|
? config.folder_id.trim()
|
|
668
690
|
: undefined;
|
|
669
691
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
670
|
-
|
|
692
|
+
let collected = 0;
|
|
671
693
|
let nextPageToken: string | undefined;
|
|
672
694
|
// The cursor for the page AFTER the one just consumed. A run that stops
|
|
673
695
|
// early persists this instead of dropping back to a full re-list — a
|
|
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
718
740
|
// start of the next page, so stopping part-way through one would skip the
|
|
719
741
|
// changes still in it.
|
|
720
742
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
721
|
-
let stoppedEarly = false;
|
|
722
743
|
for await (const items of pages) {
|
|
744
|
+
if (rejected) return null;
|
|
745
|
+
const events: EventEnvelope[] = [];
|
|
723
746
|
for (const change of items) {
|
|
724
747
|
const envelope = await this.changeToEnvelope(
|
|
725
748
|
http,
|
|
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
731
754
|
);
|
|
732
755
|
if (envelope) events.push(envelope);
|
|
733
756
|
}
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
757
|
+
collected += events.length;
|
|
758
|
+
if (!resumeCursor) {
|
|
759
|
+
// The stream is caught up: the page that carried `newStartPageToken`
|
|
760
|
+
// commits it and stamps the run. Rebuilt rather than spread over the
|
|
761
|
+
// previous checkpoint so a run that recovered from a rejected token
|
|
762
|
+
// cannot leave that token behind; without a new token the key is
|
|
763
|
+
// absent and the next run correctly re-lists.
|
|
764
|
+
await commit(events, {
|
|
765
|
+
...(nextPageToken ? { page_token: nextPageToken } : {}),
|
|
766
|
+
last_sync_at: new Date().toISOString(),
|
|
767
|
+
});
|
|
768
|
+
return {
|
|
769
|
+
status: 'complete',
|
|
770
|
+
metadata: { items_found: collected, bootstrap_complete: true },
|
|
771
|
+
};
|
|
738
772
|
}
|
|
773
|
+
// Mid-stream: commit with the cursor of the next page, a resumable
|
|
774
|
+
// position in the change stream. `last_sync_at` does NOT advance — the
|
|
775
|
+
// view-churn guard skips content for files modified before that stamp,
|
|
776
|
+
// so stamping now would make the next run treat the unread backlog as
|
|
777
|
+
// already stored and drop its content on the floor.
|
|
778
|
+
await commit(events, {
|
|
779
|
+
page_token: resumeCursor,
|
|
780
|
+
...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
|
|
781
|
+
});
|
|
782
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
739
783
|
}
|
|
740
|
-
|
|
741
784
|
if (rejected) return null;
|
|
742
785
|
|
|
743
|
-
//
|
|
744
|
-
// stream still open
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
return { events, nextPageToken: resumeCursor, partial: true };
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
return { events, nextPageToken, partial: false };
|
|
786
|
+
// Stopped on the cap or the clock, or the paginator ran out of pages
|
|
787
|
+
// (MAX_PAGES) with the stream still open. The committed cursor resumes it.
|
|
788
|
+
return {
|
|
789
|
+
status: 'more',
|
|
790
|
+
metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
|
|
791
|
+
};
|
|
753
792
|
}
|
|
754
793
|
|
|
755
794
|
private async fetchStartPageToken(http: HttpClient): Promise<string> {
|
|
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
1219
1258
|
};
|
|
1220
1259
|
}
|
|
1221
1260
|
|
|
1222
|
-
/**
|
|
1223
|
-
* Checkpoint for a bootstrap that stopped on `max_results` with pages left.
|
|
1224
|
-
*
|
|
1225
|
-
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
1226
|
-
* incremental, which is exactly the property that keeps the unread files
|
|
1227
|
-
* reachable. `bootstrap_complete: false` is surfaced in metadata so an
|
|
1228
|
-
* operator can see the traversal is still catching up rather than guessing
|
|
1229
|
-
* from a suspiciously round item count.
|
|
1230
|
-
*/
|
|
1231
|
-
private buildBootstrapResult(
|
|
1232
|
-
events: EventEnvelope[],
|
|
1233
|
-
pendingPageToken: string,
|
|
1234
|
-
listPageToken: string,
|
|
1235
|
-
startedAt: string,
|
|
1236
|
-
searchIncomplete = false
|
|
1237
|
-
): SyncResult {
|
|
1238
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1239
|
-
|
|
1240
|
-
const checkpoint: DriveCheckpoint = {
|
|
1241
|
-
pending_page_token: pendingPageToken,
|
|
1242
|
-
list_page_token: listPageToken,
|
|
1243
|
-
last_sync_at: startedAt,
|
|
1244
|
-
};
|
|
1245
|
-
|
|
1246
|
-
return {
|
|
1247
|
-
events,
|
|
1248
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
1249
|
-
metadata: {
|
|
1250
|
-
items_found: events.length,
|
|
1251
|
-
bootstrap_complete: false,
|
|
1252
|
-
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
1253
|
-
},
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
private buildResult(
|
|
1258
|
-
events: EventEnvelope[],
|
|
1259
|
-
pageToken: string | undefined,
|
|
1260
|
-
/** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
|
|
1261
|
-
keepLastSyncAt: string | undefined,
|
|
1262
|
-
flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
|
|
1263
|
-
): SyncResult {
|
|
1264
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1265
|
-
|
|
1266
|
-
// Rebuilt rather than spread over the previous checkpoint so a run that
|
|
1267
|
-
// recovered from a rejected token cannot leave that token behind. Without a
|
|
1268
|
-
// new token the key is absent and the next run correctly re-lists.
|
|
1269
|
-
const newCheckpoint: DriveCheckpoint = {
|
|
1270
|
-
...(pageToken ? { page_token: pageToken } : {}),
|
|
1271
|
-
last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
|
|
1272
|
-
};
|
|
1273
|
-
|
|
1274
|
-
return {
|
|
1275
|
-
events,
|
|
1276
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1277
|
-
metadata: {
|
|
1278
|
-
items_found: events.length,
|
|
1279
|
-
bootstrap_complete: true,
|
|
1280
|
-
...(flags.changesPending ? { changes_pending: true } : {}),
|
|
1281
|
-
...(flags.searchIncomplete ? { search_incomplete: true } : {}),
|
|
1282
|
-
},
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
1261
|
/** Overridable so a test can drive the time budget without sleeping. */
|
|
1287
1262
|
protected now(): number {
|
|
1288
1263
|
return Date.now();
|