@lobu/runtime-device 20.1.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +46 -40
- package/npm-shrinkwrap.json +283 -237
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
- package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/commands/login.js +3 -0
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/chat-user-identity.ts +41 -0
- package/vendor/cli/dist/connectors/github.ts +15 -9
- package/vendor/cli/dist/connectors/google-identity.ts +11 -0
- package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +62 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +8 -6
- package/vendor/cli/dist/connectors/reddit.ts +2 -2
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/slack-identity.ts +19 -0
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/internal/credentials.js +5 -1
- package/vendor/cli/dist/internal/oauth.d.ts +15 -1
- package/vendor/cli/dist/internal/oauth.js +9 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation-process.d.ts +32 -0
- package/vendor/connector-worker/dist/daemon/automation-process.js +102 -1
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/builtins/index.js +8 -0
- package/vendor/connector-worker/dist/daemon/builtins/os-shell.d.ts +13 -1
- package/vendor/connector-worker/dist/daemon/builtins/os-shell.js +28 -2
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +42 -51
- package/vendor/connector-worker/dist/daemon/start.js +5 -1
- package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
- package/vendor/connector-worker/dist/executor/isolate.js +45 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
- package/vendor/core/dist/contracts/worker/protocol.js +58 -9
- package/vendor/core/dist/credentials.d.ts +16 -1
- package/vendor/core/dist/credentials.js +10 -1
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +25 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -18,6 +18,18 @@
|
|
|
18
18
|
* these links authorize approvals and agent re-binding.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
/**
|
|
22
|
+
* How a platform's stored user keys are scoped for a reverse lookup.
|
|
23
|
+
*
|
|
24
|
+
* `team-prefix` — keys for one tenant all begin with `prefix`, so a search must
|
|
25
|
+
* be constrained to it (Slack: `T0XYZ:`).
|
|
26
|
+
* `global` — ids are unique platform-wide, so the namespace needs no narrowing
|
|
27
|
+
* (Google: one account can only ever have one id).
|
|
28
|
+
*/
|
|
29
|
+
export type ChatUserKeyScope =
|
|
30
|
+
| { kind: 'team-prefix'; prefix: string }
|
|
31
|
+
| { kind: 'global' };
|
|
32
|
+
|
|
21
33
|
/** How one chat platform keys a sender identity. */
|
|
22
34
|
export interface ChatUserIdentity {
|
|
23
35
|
/** Chat platform key an inbound message arrives on (`slack`, `gchat`, …). */
|
|
@@ -42,4 +54,33 @@ export interface ChatUserIdentity {
|
|
|
42
54
|
teamId: string | null | undefined,
|
|
43
55
|
platformUserId: string | null | undefined,
|
|
44
56
|
): string | null;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The REVERSE of `buildUserKey`, for "which platform id does this Lobu user
|
|
60
|
+
* have here?": how stored keys are scoped when searching within `teamId`.
|
|
61
|
+
*
|
|
62
|
+
* Required, not optional, and deliberately NOT a bare nullable string. The
|
|
63
|
+
* two failure shapes must not collapse into one value: `global` means the
|
|
64
|
+
* platform has no tenant axis so the whole namespace is in scope, while null
|
|
65
|
+
* means this platform REQUIRES a team and did not get a usable one. A caller
|
|
66
|
+
* that treats those alike widens a Slack search past its `TEAM:` prefix and
|
|
67
|
+
* can return an id from a DIFFERENT workspace — the exact cross-tenant bleed
|
|
68
|
+
* `buildUserKey` exists to prevent. Callers must refuse on null.
|
|
69
|
+
*/
|
|
70
|
+
userKeyScope(teamId: string | null | undefined): ChatUserKeyScope | null;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Recover from a stored key the id the PLATFORM addresses this user by — the
|
|
74
|
+
* form its API takes (`openDM`, a mention), which is also the form
|
|
75
|
+
* `buildUserKey` accepts back.
|
|
76
|
+
*
|
|
77
|
+
* NOT "the key minus its scope prefix". Google stores the bare account id but
|
|
78
|
+
* addresses `users/<id>`, and the chat SDK routes a DM by inferring the
|
|
79
|
+
* adapter from exactly that prefix, so a bare id does not reach Google Chat
|
|
80
|
+
* at all. Each platform owns the difference; core must never re-derive it.
|
|
81
|
+
*
|
|
82
|
+
* Returns null when the stored key is not one this platform could have
|
|
83
|
+
* written — fail closed rather than address a malformed id.
|
|
84
|
+
*/
|
|
85
|
+
platformUserIdFromKey(key: string): string | null;
|
|
45
86
|
}
|
|
@@ -905,12 +905,13 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
905
905
|
if (contentType === 'stargazers') {
|
|
906
906
|
const result = await this.syncStargazers(repo, ctx.checkpoint, token);
|
|
907
907
|
this.stampRepoAttribution(result.events, repo, { attachAutomationSignals });
|
|
908
|
+
// One commit: removed stargazers are only known against the full snapshot.
|
|
909
|
+
await ctx.commit(result.events, {
|
|
910
|
+
last_sync_at: new Date().toISOString(),
|
|
911
|
+
stargazers: result.currentStargazers,
|
|
912
|
+
} as Record<string, unknown>);
|
|
908
913
|
return {
|
|
909
|
-
|
|
910
|
-
checkpoint: {
|
|
911
|
-
last_sync_at: new Date().toISOString(),
|
|
912
|
-
stargazers: result.currentStargazers,
|
|
913
|
-
} as Record<string, unknown>,
|
|
914
|
+
status: 'complete',
|
|
914
915
|
metadata: {
|
|
915
916
|
items_found: result.events.length,
|
|
916
917
|
current_stargazers: result.currentStargazers.length,
|
|
@@ -927,11 +928,11 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
927
928
|
});
|
|
928
929
|
this.stampRepoAttribution(events, repo, { attachAutomationSignals });
|
|
929
930
|
|
|
931
|
+
await ctx.commit(events, {
|
|
932
|
+
last_sync_at: new Date().toISOString(),
|
|
933
|
+
} as Record<string, unknown>);
|
|
930
934
|
return {
|
|
931
|
-
|
|
932
|
-
checkpoint: {
|
|
933
|
-
last_sync_at: new Date().toISOString(),
|
|
934
|
-
} as Record<string, unknown>,
|
|
935
|
+
status: 'complete',
|
|
935
936
|
metadata: {
|
|
936
937
|
items_found: events.length,
|
|
937
938
|
},
|
|
@@ -2090,6 +2091,9 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2090
2091
|
query: params.query,
|
|
2091
2092
|
variables: params.variables ?? {},
|
|
2092
2093
|
},
|
|
2094
|
+
// Every caller sends a query (discussion reads), which is safe to repeat
|
|
2095
|
+
// on a 5xx even though it goes out as POST. A mutation must not set this.
|
|
2096
|
+
idempotent: true,
|
|
2093
2097
|
});
|
|
2094
2098
|
}
|
|
2095
2099
|
|
|
@@ -2101,6 +2105,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2101
2105
|
token: string | null;
|
|
2102
2106
|
body?: Record<string, unknown>;
|
|
2103
2107
|
accept?: string;
|
|
2108
|
+
idempotent?: boolean;
|
|
2104
2109
|
}): Promise<T> {
|
|
2105
2110
|
const headers: Record<string, string> = {
|
|
2106
2111
|
Accept: params.accept ?? 'application/vnd.github+json',
|
|
@@ -2115,6 +2120,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2115
2120
|
method: params.method ?? 'GET',
|
|
2116
2121
|
headers,
|
|
2117
2122
|
body: params.body ? JSON.stringify(params.body) : undefined,
|
|
2123
|
+
idempotent: params.idempotent,
|
|
2118
2124
|
});
|
|
2119
2125
|
}
|
|
2120
2126
|
}
|
|
@@ -104,4 +104,15 @@ export const gchatChatUserIdentity: ChatUserIdentity = {
|
|
|
104
104
|
buildUserKey(_teamId, platformUserId) {
|
|
105
105
|
return normalizeGoogleUserId(platformUserId);
|
|
106
106
|
},
|
|
107
|
+
// Global for the same reason `buildUserKey` ignores the team: one Google
|
|
108
|
+
// account has exactly one id, so there is no tenant axis to narrow by and
|
|
109
|
+
// nothing to bleed across. Never null — a missing team is not an error here.
|
|
110
|
+
userKeyScope: () => ({ kind: 'global' }),
|
|
111
|
+
// Re-wraps the bare stored id as the resource name Chat addresses people by.
|
|
112
|
+
// The `users/` prefix is load-bearing on the way out as well as in: the chat
|
|
113
|
+
// SDK picks the Google Chat adapter off it, so a bare id is not routable.
|
|
114
|
+
platformUserIdFromKey(key) {
|
|
115
|
+
const bare = normalizeGoogleUserId(key);
|
|
116
|
+
return bare ? `${GCHAT_USER_PREFIX}${bare}` : null;
|
|
117
|
+
},
|
|
107
118
|
};
|
|
@@ -457,7 +457,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
457
457
|
const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
|
|
458
458
|
|
|
459
459
|
const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
|
|
460
|
-
const events: EventEnvelope[] = [];
|
|
461
460
|
const durableChanges = ctx.feedKey === 'changes';
|
|
462
461
|
|
|
463
462
|
// A checkpoint minted under a different scope (or by an older version that
|
|
@@ -467,10 +466,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
467
466
|
const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
|
|
468
467
|
const resumable = checkpoint.scope === scope;
|
|
469
468
|
|
|
470
|
-
/** Traversal reached the last page:
|
|
471
|
-
const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
|
|
472
|
-
|
|
473
|
-
|
|
469
|
+
/** Traversal reached the last page: commit it with its cursor and stamp the run. */
|
|
470
|
+
const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
|
|
471
|
+
await ctx.commit(items, {
|
|
472
|
+
...this.completedCheckpoint(syncToken),
|
|
473
|
+
scope,
|
|
474
|
+
} as Record<string, unknown>);
|
|
475
|
+
return { status: 'complete' };
|
|
474
476
|
};
|
|
475
477
|
|
|
476
478
|
if (resumable && checkpoint.sync_token) {
|
|
@@ -517,15 +519,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
517
519
|
const baseParams = params.toString();
|
|
518
520
|
|
|
519
521
|
/**
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
522
|
+
* Mid-traversal: the next page token, with no sync token and no
|
|
523
|
+
* `last_sync_at`, so nothing downstream can read it as a completed window.
|
|
524
|
+
* A run that stops here resumes from `token`.
|
|
523
525
|
*/
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
checkpoint: { scope, pending: { params: baseParams, page_token: token } },
|
|
527
|
-
});
|
|
526
|
+
const parked = (token: string) =>
|
|
527
|
+
({ scope, pending: { params: baseParams, page_token: token } }) as Record<string, unknown>;
|
|
528
528
|
|
|
529
|
+
let collected = 0;
|
|
529
530
|
let pageToken = pending?.page_token;
|
|
530
531
|
const seenTokens = new Set<string>();
|
|
531
532
|
for (let page = 0; ; page++) {
|
|
@@ -549,6 +550,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
549
550
|
// A provider page is atomic: `max_results` caps how much a run starts,
|
|
550
551
|
// never how much of a fetched page is stored. Dropping a page's tail
|
|
551
552
|
// would lose those events for good once the cursor moved past them.
|
|
553
|
+
const events: EventEnvelope[] = [];
|
|
552
554
|
for (const calEvent of items) {
|
|
553
555
|
const envelope = durableChanges
|
|
554
556
|
? this.calendarEventToChangeEnvelope(calEvent)
|
|
@@ -570,17 +572,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
570
572
|
throw new Error('Google Calendar returned a repeated page token.');
|
|
571
573
|
}
|
|
572
574
|
pageToken = data.nextPageToken;
|
|
575
|
+
await ctx.commit(events, parked(pageToken));
|
|
576
|
+
collected += events.length;
|
|
573
577
|
|
|
574
578
|
// Stop once this run has collected its share, once the provider returns
|
|
575
579
|
// an empty page, or at the hard ceiling — the last of which is the only
|
|
576
580
|
// bound on a window of nothing but filtered-out items, where
|
|
577
|
-
// `
|
|
581
|
+
// `collected` never reaches the cap. The traversal continues next run.
|
|
578
582
|
if (
|
|
579
|
-
|
|
583
|
+
collected >= maxResults ||
|
|
580
584
|
items.length === 0 ||
|
|
581
585
|
page + 1 >= MAX_SYNC_PAGES
|
|
582
586
|
) {
|
|
583
|
-
return
|
|
587
|
+
return { status: 'more' };
|
|
584
588
|
}
|
|
585
589
|
}
|
|
586
590
|
}
|
|
@@ -985,29 +989,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
985
989
|
};
|
|
986
990
|
}
|
|
987
991
|
|
|
988
|
-
private
|
|
989
|
-
events: EventEnvelope[],
|
|
990
|
-
syncToken: string | undefined
|
|
991
|
-
): SyncResult {
|
|
992
|
-
// Sort events by occurred_at descending
|
|
993
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
994
|
-
|
|
992
|
+
private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
|
|
995
993
|
// Built fresh rather than spread over the previous checkpoint: the whole
|
|
996
994
|
// object replaces the stored one, so a run that recovered from a rejected
|
|
997
995
|
// token cannot leave that token behind. When the sync produced no new token
|
|
998
996
|
// the key is absent, and the next run correctly starts from a full sync.
|
|
999
|
-
|
|
997
|
+
return {
|
|
1000
998
|
...(syncToken ? { sync_token: syncToken } : {}),
|
|
1001
999
|
last_sync_at: new Date().toISOString(),
|
|
1002
1000
|
};
|
|
1003
|
-
|
|
1004
|
-
return {
|
|
1005
|
-
events,
|
|
1006
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1007
|
-
metadata: {
|
|
1008
|
-
items_found: events.length,
|
|
1009
|
-
},
|
|
1010
|
-
};
|
|
1011
1001
|
}
|
|
1012
1002
|
|
|
1013
1003
|
// Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
|
|
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
|
|
|
286
286
|
// Connector
|
|
287
287
|
// ---------------------------------------------------------------------------
|
|
288
288
|
|
|
289
|
+
type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Checkpoint for a bootstrap with list pages still unread.
|
|
293
|
+
*
|
|
294
|
+
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
295
|
+
* incremental, which is exactly the property that keeps the unread files
|
|
296
|
+
* reachable.
|
|
297
|
+
*/
|
|
298
|
+
function bootstrapCheckpoint(
|
|
299
|
+
pendingPageToken: string,
|
|
300
|
+
listPageToken: string,
|
|
301
|
+
startedAt: string
|
|
302
|
+
): DriveCheckpoint {
|
|
303
|
+
return {
|
|
304
|
+
pending_page_token: pendingPageToken,
|
|
305
|
+
list_page_token: listPageToken,
|
|
306
|
+
last_sync_at: startedAt,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
289
310
|
export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
290
311
|
Record<string, unknown>,
|
|
291
312
|
DriveConfig
|
|
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
493
514
|
|
|
494
515
|
// An unfinished bootstrap outranks everything: going incremental with pages
|
|
495
516
|
// still unread would strand every file the traversal never reached.
|
|
517
|
+
const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
|
|
518
|
+
ctx.commit(events, next as Record<string, unknown>);
|
|
519
|
+
|
|
496
520
|
if (checkpoint.list_page_token && checkpoint.pending_page_token) {
|
|
497
|
-
return this.fullSync(http, config, {
|
|
521
|
+
return this.fullSync(http, config, commit, {
|
|
498
522
|
startToken: checkpoint.pending_page_token,
|
|
499
523
|
listPageToken: checkpoint.list_page_token,
|
|
500
524
|
startedAt: checkpoint.last_sync_at,
|
|
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
506
530
|
http,
|
|
507
531
|
checkpoint.page_token,
|
|
508
532
|
config,
|
|
533
|
+
commit,
|
|
509
534
|
checkpoint.last_sync_at
|
|
510
535
|
);
|
|
511
|
-
if (result)
|
|
512
|
-
// A partial run must NOT advance `last_sync_at`. The view-churn guard
|
|
513
|
-
// skips content for files modified before that stamp, so stamping now
|
|
514
|
-
// would make the next run treat the unread backlog as already-stored
|
|
515
|
-
// and drop its content on the floor.
|
|
516
|
-
return this.buildResult(
|
|
517
|
-
result.events,
|
|
518
|
-
result.nextPageToken,
|
|
519
|
-
result.partial ? checkpoint.last_sync_at : undefined,
|
|
520
|
-
{ changesPending: result.partial }
|
|
521
|
-
);
|
|
522
|
-
}
|
|
536
|
+
if (result) return result;
|
|
523
537
|
// Token rejected (see isPageTokenRejection). Fall through to ONE full
|
|
524
538
|
// re-list, exactly as the Calendar connector does — never a retry loop.
|
|
525
539
|
}
|
|
526
540
|
|
|
527
|
-
return this.fullSync(http, config);
|
|
541
|
+
return this.fullSync(http, config, commit);
|
|
528
542
|
}
|
|
529
543
|
|
|
530
544
|
/**
|
|
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
539
553
|
private async fullSync(
|
|
540
554
|
http: HttpClient,
|
|
541
555
|
config: DriveConfig,
|
|
556
|
+
commit: DriveCommit,
|
|
542
557
|
resume?: { startToken: string; listPageToken: string; startedAt?: string }
|
|
543
558
|
): Promise<SyncResult> {
|
|
544
559
|
// A resumed bootstrap keeps the token minted when the traversal STARTED —
|
|
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
555
570
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
556
571
|
const includeContent = config.include_content !== false;
|
|
557
572
|
const q = buildListQuery(config);
|
|
558
|
-
|
|
573
|
+
let collected = 0;
|
|
559
574
|
|
|
560
575
|
// 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
|
|
561
576
|
// configurable max_results. Defensive bound against a self-referential
|
|
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
605
620
|
{ maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
|
|
606
621
|
);
|
|
607
622
|
|
|
623
|
+
// A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
|
|
624
|
+
// client-side against a change record, so the only way to keep honouring it
|
|
625
|
+
// is to keep listing. Withholding the token re-lists next run; promoting it
|
|
626
|
+
// would silently widen the feed to every changed file in the Drive.
|
|
627
|
+
const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
|
|
628
|
+
|
|
608
629
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
609
630
|
for await (const items of pages) {
|
|
631
|
+
const events: EventEnvelope[] = [];
|
|
610
632
|
for (const file of items) {
|
|
611
633
|
events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
|
|
612
634
|
}
|
|
613
|
-
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
635
|
+
collected += events.length;
|
|
636
|
+
// Only Drive withholding a nextPageToken proves the traversal exhausted,
|
|
637
|
+
// so that — not the reason a run stops — is what may promote the change
|
|
638
|
+
// token. Until then each page commits with the bootstrap parked on the
|
|
639
|
+
// next unread page: a page is consumed whole or not at all, so the list
|
|
640
|
+
// token always addresses the first file not yet stored.
|
|
641
|
+
if (!nextListPageToken) {
|
|
642
|
+
await commit(events, {
|
|
643
|
+
...(hasCustomQuery ? {} : { page_token: startToken }),
|
|
644
|
+
last_sync_at: bootstrapStartedAt,
|
|
645
|
+
});
|
|
646
|
+
return {
|
|
647
|
+
status: 'complete',
|
|
648
|
+
metadata: {
|
|
649
|
+
items_found: collected,
|
|
650
|
+
bootstrap_complete: true,
|
|
651
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
652
|
+
},
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
|
|
656
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
632
657
|
}
|
|
633
658
|
|
|
634
|
-
//
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
659
|
+
// Unread pages remain, and WHY this run stopped is irrelevant: the item
|
|
660
|
+
// cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
|
|
661
|
+
// the change feed will never replay. The bootstrap resumes next run.
|
|
662
|
+
return {
|
|
663
|
+
status: 'more',
|
|
664
|
+
metadata: {
|
|
665
|
+
items_found: collected,
|
|
666
|
+
bootstrap_complete: false,
|
|
667
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
668
|
+
},
|
|
669
|
+
};
|
|
645
670
|
}
|
|
646
671
|
|
|
647
672
|
/**
|
|
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
654
679
|
http: HttpClient,
|
|
655
680
|
pageToken: string,
|
|
656
681
|
config: DriveConfig,
|
|
682
|
+
commit: DriveCommit,
|
|
657
683
|
lastSyncAt?: string
|
|
658
|
-
): Promise<{
|
|
659
|
-
events: EventEnvelope[];
|
|
660
|
-
nextPageToken?: string;
|
|
661
|
-
partial: boolean;
|
|
662
|
-
} | null> {
|
|
684
|
+
): Promise<SyncResult | null> {
|
|
663
685
|
const includeContent = config.include_content !== false;
|
|
664
686
|
const includeTrashed = Boolean(config.include_trashed);
|
|
665
687
|
const folderId =
|
|
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
667
689
|
? config.folder_id.trim()
|
|
668
690
|
: undefined;
|
|
669
691
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
670
|
-
|
|
692
|
+
let collected = 0;
|
|
671
693
|
let nextPageToken: string | undefined;
|
|
672
694
|
// The cursor for the page AFTER the one just consumed. A run that stops
|
|
673
695
|
// early persists this instead of dropping back to a full re-list — a
|
|
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
718
740
|
// start of the next page, so stopping part-way through one would skip the
|
|
719
741
|
// changes still in it.
|
|
720
742
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
721
|
-
let stoppedEarly = false;
|
|
722
743
|
for await (const items of pages) {
|
|
744
|
+
if (rejected) return null;
|
|
745
|
+
const events: EventEnvelope[] = [];
|
|
723
746
|
for (const change of items) {
|
|
724
747
|
const envelope = await this.changeToEnvelope(
|
|
725
748
|
http,
|
|
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
731
754
|
);
|
|
732
755
|
if (envelope) events.push(envelope);
|
|
733
756
|
}
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
757
|
+
collected += events.length;
|
|
758
|
+
if (!resumeCursor) {
|
|
759
|
+
// The stream is caught up: the page that carried `newStartPageToken`
|
|
760
|
+
// commits it and stamps the run. Rebuilt rather than spread over the
|
|
761
|
+
// previous checkpoint so a run that recovered from a rejected token
|
|
762
|
+
// cannot leave that token behind; without a new token the key is
|
|
763
|
+
// absent and the next run correctly re-lists.
|
|
764
|
+
await commit(events, {
|
|
765
|
+
...(nextPageToken ? { page_token: nextPageToken } : {}),
|
|
766
|
+
last_sync_at: new Date().toISOString(),
|
|
767
|
+
});
|
|
768
|
+
return {
|
|
769
|
+
status: 'complete',
|
|
770
|
+
metadata: { items_found: collected, bootstrap_complete: true },
|
|
771
|
+
};
|
|
738
772
|
}
|
|
773
|
+
// Mid-stream: commit with the cursor of the next page, a resumable
|
|
774
|
+
// position in the change stream. `last_sync_at` does NOT advance — the
|
|
775
|
+
// view-churn guard skips content for files modified before that stamp,
|
|
776
|
+
// so stamping now would make the next run treat the unread backlog as
|
|
777
|
+
// already stored and drop its content on the floor.
|
|
778
|
+
await commit(events, {
|
|
779
|
+
page_token: resumeCursor,
|
|
780
|
+
...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
|
|
781
|
+
});
|
|
782
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
739
783
|
}
|
|
740
|
-
|
|
741
784
|
if (rejected) return null;
|
|
742
785
|
|
|
743
|
-
//
|
|
744
|
-
// stream still open
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
return { events, nextPageToken: resumeCursor, partial: true };
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
return { events, nextPageToken, partial: false };
|
|
786
|
+
// Stopped on the cap or the clock, or the paginator ran out of pages
|
|
787
|
+
// (MAX_PAGES) with the stream still open. The committed cursor resumes it.
|
|
788
|
+
return {
|
|
789
|
+
status: 'more',
|
|
790
|
+
metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
|
|
791
|
+
};
|
|
753
792
|
}
|
|
754
793
|
|
|
755
794
|
private async fetchStartPageToken(http: HttpClient): Promise<string> {
|
|
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
1219
1258
|
};
|
|
1220
1259
|
}
|
|
1221
1260
|
|
|
1222
|
-
/**
|
|
1223
|
-
* Checkpoint for a bootstrap that stopped on `max_results` with pages left.
|
|
1224
|
-
*
|
|
1225
|
-
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
1226
|
-
* incremental, which is exactly the property that keeps the unread files
|
|
1227
|
-
* reachable. `bootstrap_complete: false` is surfaced in metadata so an
|
|
1228
|
-
* operator can see the traversal is still catching up rather than guessing
|
|
1229
|
-
* from a suspiciously round item count.
|
|
1230
|
-
*/
|
|
1231
|
-
private buildBootstrapResult(
|
|
1232
|
-
events: EventEnvelope[],
|
|
1233
|
-
pendingPageToken: string,
|
|
1234
|
-
listPageToken: string,
|
|
1235
|
-
startedAt: string,
|
|
1236
|
-
searchIncomplete = false
|
|
1237
|
-
): SyncResult {
|
|
1238
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1239
|
-
|
|
1240
|
-
const checkpoint: DriveCheckpoint = {
|
|
1241
|
-
pending_page_token: pendingPageToken,
|
|
1242
|
-
list_page_token: listPageToken,
|
|
1243
|
-
last_sync_at: startedAt,
|
|
1244
|
-
};
|
|
1245
|
-
|
|
1246
|
-
return {
|
|
1247
|
-
events,
|
|
1248
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
1249
|
-
metadata: {
|
|
1250
|
-
items_found: events.length,
|
|
1251
|
-
bootstrap_complete: false,
|
|
1252
|
-
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
1253
|
-
},
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
private buildResult(
|
|
1258
|
-
events: EventEnvelope[],
|
|
1259
|
-
pageToken: string | undefined,
|
|
1260
|
-
/** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
|
|
1261
|
-
keepLastSyncAt: string | undefined,
|
|
1262
|
-
flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
|
|
1263
|
-
): SyncResult {
|
|
1264
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1265
|
-
|
|
1266
|
-
// Rebuilt rather than spread over the previous checkpoint so a run that
|
|
1267
|
-
// recovered from a rejected token cannot leave that token behind. Without a
|
|
1268
|
-
// new token the key is absent and the next run correctly re-lists.
|
|
1269
|
-
const newCheckpoint: DriveCheckpoint = {
|
|
1270
|
-
...(pageToken ? { page_token: pageToken } : {}),
|
|
1271
|
-
last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
|
|
1272
|
-
};
|
|
1273
|
-
|
|
1274
|
-
return {
|
|
1275
|
-
events,
|
|
1276
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1277
|
-
metadata: {
|
|
1278
|
-
items_found: events.length,
|
|
1279
|
-
bootstrap_complete: true,
|
|
1280
|
-
...(flags.changesPending ? { changes_pending: true } : {}),
|
|
1281
|
-
...(flags.searchIncomplete ? { search_incomplete: true } : {}),
|
|
1282
|
-
},
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
1261
|
/** Overridable so a test can drive the time budget without sleeping. */
|
|
1287
1262
|
protected now(): number {
|
|
1288
1263
|
return Date.now();
|