@opengeni/worker-bundle 0.16.21 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +145 -32
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +3 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -7
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +2 -2
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +129 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +2 -2
- package/dist/activities/run-input.d.ts +5 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +11 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -147
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +675 -16460
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +40 -7
- package/dist/sandbox-routing.d.ts +13 -6
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1044 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1905 -559
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +16 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +47 -6
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +52 -60
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +1317 -0
- package/src/activities/rig-verification.ts +567 -77
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +40 -10
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +96 -7
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +77 -13
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +50 -118
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +202 -7
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +331 -157
- package/src/sandbox-routing.ts +49 -39
- package/src/worker-service-lifecycle.ts +82 -12
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +6 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
- package/src/activities/toolspace-token-renewal.ts +0 -57
|
@@ -16,7 +16,7 @@ export type GitCredentialRenewalResult = {
|
|
|
16
16
|
export type GitCredentialRenewalFailure = {
|
|
17
17
|
providers: readonly GitCredentialProvider[];
|
|
18
18
|
retryDelayMs: number;
|
|
19
|
-
errorClass:
|
|
19
|
+
errorClass: "GitCredentialRenewalOperationError";
|
|
20
20
|
};
|
|
21
21
|
|
|
22
22
|
export type GitCredentialRenewalController = {
|
|
@@ -133,7 +133,7 @@ export function startGitCredentialRenewalLoop(
|
|
|
133
133
|
// Observability callbacks never own credential liveness.
|
|
134
134
|
}
|
|
135
135
|
scheduleRefresh(nextDelayMs);
|
|
136
|
-
} catch
|
|
136
|
+
} catch {
|
|
137
137
|
if (stopped) return;
|
|
138
138
|
const scheduledRetryMs = retryDelayMs;
|
|
139
139
|
retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
|
|
@@ -141,7 +141,7 @@ export function startGitCredentialRenewalLoop(
|
|
|
141
141
|
options.onFailure?.({
|
|
142
142
|
providers,
|
|
143
143
|
retryDelayMs: scheduledRetryMs,
|
|
144
|
-
errorClass:
|
|
144
|
+
errorClass: "GitCredentialRenewalOperationError",
|
|
145
145
|
});
|
|
146
146
|
} catch {
|
|
147
147
|
// Observability callbacks never own credential liveness.
|
package/src/activities/goals.ts
CHANGED
|
@@ -24,9 +24,13 @@ import {
|
|
|
24
24
|
sumUsageQuantity,
|
|
25
25
|
type Database,
|
|
26
26
|
} from "@opengeni/db";
|
|
27
|
-
import type {
|
|
27
|
+
import type {
|
|
28
|
+
ControlActivityServices,
|
|
29
|
+
MaybeContinueGoalInput,
|
|
30
|
+
MaybeContinueGoalResult,
|
|
31
|
+
} from "./types";
|
|
28
32
|
|
|
29
|
-
export function createGoalActivities(services: () => Promise<
|
|
33
|
+
export function createGoalActivities(services: () => Promise<ControlActivityServices>) {
|
|
30
34
|
async function enqueueGoalRetryWake(input: MaybeContinueGoalInput): Promise<void> {
|
|
31
35
|
const { db } = await services();
|
|
32
36
|
await enqueueSessionWorkflowWakeIfRunnable(db, {
|
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import type { GoogleDriveSelectedSource } from "@opengeni/contracts/google-drive";
|
|
3
|
+
import {
|
|
4
|
+
GOOGLE_DRIVE_FOLDER_MIME_TYPE,
|
|
5
|
+
planGoogleDriveTransfer,
|
|
6
|
+
type GoogleDriveInventoryEntry,
|
|
7
|
+
type GoogleDriveInventoryProviderItem,
|
|
8
|
+
} from "@opengeni/documents/google-drive";
|
|
9
|
+
|
|
10
|
+
const MAX_TOKEN_CHARS = 4096;
|
|
11
|
+
const MAX_BOUNDARY_ID_CHARS = 1024;
|
|
12
|
+
const MAX_ANCESTOR_DEPTH = 100;
|
|
13
|
+
|
|
14
|
+
export type GoogleDriveSyncBudget = {
|
|
15
|
+
examinedItems: number;
|
|
16
|
+
providerRequests: number;
|
|
17
|
+
elapsedMs: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export type GoogleDriveChangesCursor = {
|
|
21
|
+
version: 1;
|
|
22
|
+
kind: "google_drive_changes";
|
|
23
|
+
connectionId: string;
|
|
24
|
+
googlePermissionId: string;
|
|
25
|
+
sourceId: string;
|
|
26
|
+
driveId: string | null;
|
|
27
|
+
boundaryId: string;
|
|
28
|
+
pageToken: string;
|
|
29
|
+
cursorGeneration: number;
|
|
30
|
+
lastFullReconciliationAt: string;
|
|
31
|
+
nextFullReconciliationAt: string;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type GoogleDriveChangesCheckpoint = {
|
|
35
|
+
version: 2;
|
|
36
|
+
pageToken: string;
|
|
37
|
+
requiresFullReconciliation: boolean;
|
|
38
|
+
seenPageTokenHashes: string[];
|
|
39
|
+
budget: GoogleDriveSyncBudget;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export type GoogleDriveChange = {
|
|
43
|
+
fileId: string;
|
|
44
|
+
removed: boolean;
|
|
45
|
+
file: GoogleDriveInventoryProviderItem | null;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type GoogleDriveChangesPage = {
|
|
49
|
+
changes: GoogleDriveChange[];
|
|
50
|
+
nextPageToken: string | null;
|
|
51
|
+
newStartPageToken: string | null;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export type GoogleDriveChangesStopReason =
|
|
55
|
+
| "item_limit"
|
|
56
|
+
| "api_request_limit"
|
|
57
|
+
| "elapsed_time_limit";
|
|
58
|
+
|
|
59
|
+
export type GoogleDriveChangesDrainResult = {
|
|
60
|
+
status: "complete" | "paused";
|
|
61
|
+
stopReason: GoogleDriveChangesStopReason | null;
|
|
62
|
+
entries: GoogleDriveInventoryEntry[];
|
|
63
|
+
checkpoint: GoogleDriveChangesCheckpoint | null;
|
|
64
|
+
newStartPageToken: string | null;
|
|
65
|
+
requiresFullReconciliation: boolean;
|
|
66
|
+
providerRequests: number;
|
|
67
|
+
elapsedMs: number;
|
|
68
|
+
budget: GoogleDriveSyncBudget;
|
|
69
|
+
hardLimitReached: boolean;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
export function parseGoogleDriveChangesCursor(
|
|
73
|
+
value: Record<string, unknown> | null,
|
|
74
|
+
expected: {
|
|
75
|
+
connectionId: string;
|
|
76
|
+
googlePermissionId: string;
|
|
77
|
+
sourceId: string;
|
|
78
|
+
driveId: string | null;
|
|
79
|
+
},
|
|
80
|
+
): GoogleDriveChangesCursor | null {
|
|
81
|
+
if (!value || value.version !== 1 || value.kind !== "google_drive_changes") return null;
|
|
82
|
+
if (
|
|
83
|
+
value.connectionId !== expected.connectionId ||
|
|
84
|
+
value.googlePermissionId !== expected.googlePermissionId ||
|
|
85
|
+
value.sourceId !== expected.sourceId ||
|
|
86
|
+
value.driveId !== expected.driveId ||
|
|
87
|
+
typeof value.boundaryId !== "string" ||
|
|
88
|
+
value.boundaryId.length < 1 ||
|
|
89
|
+
value.boundaryId.length > MAX_BOUNDARY_ID_CHARS ||
|
|
90
|
+
typeof value.pageToken !== "string" ||
|
|
91
|
+
value.pageToken.length < 1 ||
|
|
92
|
+
value.pageToken.length > MAX_TOKEN_CHARS ||
|
|
93
|
+
typeof value.cursorGeneration !== "number" ||
|
|
94
|
+
!Number.isSafeInteger(value.cursorGeneration) ||
|
|
95
|
+
value.cursorGeneration < 1 ||
|
|
96
|
+
typeof value.lastFullReconciliationAt !== "string" ||
|
|
97
|
+
!Number.isFinite(Date.parse(value.lastFullReconciliationAt)) ||
|
|
98
|
+
typeof value.nextFullReconciliationAt !== "string" ||
|
|
99
|
+
!Number.isFinite(Date.parse(value.nextFullReconciliationAt))
|
|
100
|
+
) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
return value as GoogleDriveChangesCursor;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function buildGoogleDriveChangesCursor(input: {
|
|
107
|
+
connectionId: string;
|
|
108
|
+
googlePermissionId: string;
|
|
109
|
+
sourceId: string;
|
|
110
|
+
driveId: string | null;
|
|
111
|
+
boundaryId: string;
|
|
112
|
+
pageToken: string;
|
|
113
|
+
previousGeneration: number;
|
|
114
|
+
reconciledAt: Date;
|
|
115
|
+
fullReconciliationIntervalMs: number;
|
|
116
|
+
}): GoogleDriveChangesCursor {
|
|
117
|
+
return {
|
|
118
|
+
version: 1,
|
|
119
|
+
kind: "google_drive_changes",
|
|
120
|
+
connectionId: input.connectionId,
|
|
121
|
+
googlePermissionId: input.googlePermissionId,
|
|
122
|
+
sourceId: input.sourceId,
|
|
123
|
+
driveId: input.driveId,
|
|
124
|
+
boundaryId: boundedBoundaryId(input.boundaryId),
|
|
125
|
+
pageToken: boundedToken(input.pageToken),
|
|
126
|
+
cursorGeneration: input.previousGeneration + 1,
|
|
127
|
+
lastFullReconciliationAt: input.reconciledAt.toISOString(),
|
|
128
|
+
nextFullReconciliationAt: new Date(
|
|
129
|
+
input.reconciledAt.getTime() + input.fullReconciliationIntervalMs,
|
|
130
|
+
).toISOString(),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function advanceGoogleDriveChangesCursor(
|
|
135
|
+
cursor: GoogleDriveChangesCursor,
|
|
136
|
+
pageToken: string,
|
|
137
|
+
): GoogleDriveChangesCursor {
|
|
138
|
+
return {
|
|
139
|
+
...cursor,
|
|
140
|
+
pageToken: boundedToken(pageToken),
|
|
141
|
+
cursorGeneration: cursor.cursorGeneration + 1,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function googleDriveFullReconciliationDue(cursor: GoogleDriveChangesCursor, now: Date) {
|
|
146
|
+
return Date.parse(cursor.nextFullReconciliationAt) <= now.getTime();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export async function resolveGoogleDriveChangesBoundaryId(input: {
|
|
150
|
+
source: GoogleDriveSelectedSource;
|
|
151
|
+
getFile: (fileId: string) => Promise<GoogleDriveInventoryProviderItem | null>;
|
|
152
|
+
}): Promise<string> {
|
|
153
|
+
if (input.source.id !== "root" || input.source.driveId !== null) {
|
|
154
|
+
return boundedBoundaryId(input.source.id);
|
|
155
|
+
}
|
|
156
|
+
const root = await input.getFile("root");
|
|
157
|
+
if (!root || root.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE || root.trashed) {
|
|
158
|
+
throw new GoogleDriveChangesProtocolError("google_drive_root_boundary_invalid");
|
|
159
|
+
}
|
|
160
|
+
return boundedBoundaryId(root.id);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function parseGoogleDriveChangesCheckpoint(
|
|
164
|
+
value: unknown,
|
|
165
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
166
|
+
): GoogleDriveChangesCheckpoint | null {
|
|
167
|
+
try {
|
|
168
|
+
return validatedCheckpoint(value, limits);
|
|
169
|
+
} catch {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export async function drainGoogleDriveChanges(input: {
|
|
175
|
+
source: GoogleDriveSelectedSource;
|
|
176
|
+
cursor: GoogleDriveChangesCursor;
|
|
177
|
+
checkpoint: GoogleDriveChangesCheckpoint | null;
|
|
178
|
+
maxItems: number;
|
|
179
|
+
maxProviderRequests: number;
|
|
180
|
+
maxElapsedMs: number;
|
|
181
|
+
maxInvocationElapsedMs: number;
|
|
182
|
+
maxFileBytes: number;
|
|
183
|
+
listChanges: (pageToken: string, pageSize: number) => Promise<GoogleDriveChangesPage>;
|
|
184
|
+
getFile: (fileId: string) => Promise<GoogleDriveInventoryProviderItem | null>;
|
|
185
|
+
observedExternalObjectIds: (ids: string[]) => Promise<Set<string>>;
|
|
186
|
+
now?: () => number;
|
|
187
|
+
}): Promise<GoogleDriveChangesDrainResult> {
|
|
188
|
+
const now = input.now ?? Date.now;
|
|
189
|
+
const startedAt = now();
|
|
190
|
+
const checkpoint = input.checkpoint ? validatedCheckpoint(input.checkpoint, input) : null;
|
|
191
|
+
let pageToken = boundedToken(checkpoint?.pageToken ?? input.cursor.pageToken);
|
|
192
|
+
let requiresFullReconciliation = checkpoint?.requiresFullReconciliation ?? false;
|
|
193
|
+
const seenPageTokenHashes = new Set(checkpoint?.seenPageTokenHashes ?? []);
|
|
194
|
+
const initialBudget = checkpoint?.budget ?? emptyBudget();
|
|
195
|
+
const budget = { ...initialBudget };
|
|
196
|
+
const entries = new Map<string, GoogleDriveInventoryEntry>();
|
|
197
|
+
|
|
198
|
+
while (true) {
|
|
199
|
+
const invocationElapsedMs = Math.max(0, now() - startedAt);
|
|
200
|
+
budget.elapsedMs = initialBudget.elapsedMs + invocationElapsedMs;
|
|
201
|
+
const stopReason = limitReason(budget, input, invocationElapsedMs);
|
|
202
|
+
if (stopReason) {
|
|
203
|
+
return {
|
|
204
|
+
status: "paused",
|
|
205
|
+
stopReason,
|
|
206
|
+
entries: [...entries.values()],
|
|
207
|
+
checkpoint: {
|
|
208
|
+
version: 2,
|
|
209
|
+
pageToken,
|
|
210
|
+
requiresFullReconciliation,
|
|
211
|
+
seenPageTokenHashes: [...seenPageTokenHashes],
|
|
212
|
+
budget: { ...budget },
|
|
213
|
+
},
|
|
214
|
+
newStartPageToken: null,
|
|
215
|
+
requiresFullReconciliation,
|
|
216
|
+
providerRequests: budget.providerRequests,
|
|
217
|
+
elapsedMs: budget.elapsedMs,
|
|
218
|
+
budget: { ...budget },
|
|
219
|
+
hardLimitReached:
|
|
220
|
+
stopReason !== "elapsed_time_limit" || budget.elapsedMs >= input.maxElapsedMs,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const pageTokenHash = createHash("sha256").update(pageToken).digest("hex");
|
|
225
|
+
if (seenPageTokenHashes.has(pageTokenHash)) {
|
|
226
|
+
throw new GoogleDriveChangesProtocolError("google_drive_changes_page_token_cycle");
|
|
227
|
+
}
|
|
228
|
+
seenPageTokenHashes.add(pageTokenHash);
|
|
229
|
+
budget.providerRequests += 1;
|
|
230
|
+
const pageSize = Math.min(100, input.maxItems - budget.examinedItems);
|
|
231
|
+
let page: GoogleDriveChangesPage;
|
|
232
|
+
try {
|
|
233
|
+
page = await input.listChanges(pageToken, pageSize);
|
|
234
|
+
} catch (error) {
|
|
235
|
+
budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
|
|
236
|
+
if (error instanceof GoogleDriveCursorInvalidError) error.budget = { ...budget };
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
if (page.changes.length > pageSize) {
|
|
240
|
+
throw new GoogleDriveChangesProtocolError("google_drive_changes_page_exceeds_item_budget");
|
|
241
|
+
}
|
|
242
|
+
budget.examinedItems += page.changes.length;
|
|
243
|
+
|
|
244
|
+
const repairCandidates = new Set<string>();
|
|
245
|
+
for (const change of page.changes) {
|
|
246
|
+
if (change.removed || !change.file || change.file.trashed) {
|
|
247
|
+
repairCandidates.add(change.fileId);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
const membership = await isInsideSource({
|
|
251
|
+
file: change.file,
|
|
252
|
+
boundaryId: input.cursor.boundaryId,
|
|
253
|
+
getFile: async (fileId) => {
|
|
254
|
+
const ancestryElapsedMs = Math.max(0, now() - startedAt);
|
|
255
|
+
budget.elapsedMs = initialBudget.elapsedMs + ancestryElapsedMs;
|
|
256
|
+
if (
|
|
257
|
+
budget.providerRequests >= input.maxProviderRequests ||
|
|
258
|
+
budget.elapsedMs >= input.maxElapsedMs ||
|
|
259
|
+
ancestryElapsedMs >= input.maxInvocationElapsedMs
|
|
260
|
+
) {
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
budget.providerRequests += 1;
|
|
264
|
+
const parent = await input.getFile(fileId);
|
|
265
|
+
budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
|
|
266
|
+
return parent;
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
if (change.file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
|
|
270
|
+
entries.delete(change.fileId);
|
|
271
|
+
if (membership !== "outside") requiresFullReconciliation = true;
|
|
272
|
+
else repairCandidates.add(change.fileId);
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
if (membership === "inside") {
|
|
276
|
+
entries.set(change.fileId, toInventoryEntry(change.file, input.source, input.maxFileBytes));
|
|
277
|
+
} else if (membership === "unknown") {
|
|
278
|
+
entries.delete(change.fileId);
|
|
279
|
+
requiresFullReconciliation = true;
|
|
280
|
+
} else {
|
|
281
|
+
repairCandidates.add(change.fileId);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (repairCandidates.size > 0) {
|
|
285
|
+
for (const fileId of repairCandidates) {
|
|
286
|
+
if (entries.delete(fileId)) requiresFullReconciliation = true;
|
|
287
|
+
}
|
|
288
|
+
const observed = await input.observedExternalObjectIds([...repairCandidates]);
|
|
289
|
+
if (observed.size > 0) requiresFullReconciliation = true;
|
|
290
|
+
}
|
|
291
|
+
if (page.nextPageToken) {
|
|
292
|
+
pageToken = boundedToken(page.nextPageToken);
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
if (!page.newStartPageToken) {
|
|
296
|
+
throw new GoogleDriveChangesProtocolError("google_drive_changes_terminal_token_missing");
|
|
297
|
+
}
|
|
298
|
+
budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
|
|
299
|
+
return {
|
|
300
|
+
status: "complete",
|
|
301
|
+
stopReason: null,
|
|
302
|
+
entries: [...entries.values()],
|
|
303
|
+
checkpoint: null,
|
|
304
|
+
newStartPageToken: boundedToken(page.newStartPageToken),
|
|
305
|
+
requiresFullReconciliation,
|
|
306
|
+
providerRequests: budget.providerRequests,
|
|
307
|
+
elapsedMs: budget.elapsedMs,
|
|
308
|
+
budget: { ...budget },
|
|
309
|
+
hardLimitReached: false,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function isInsideSource(input: {
|
|
315
|
+
file: GoogleDriveInventoryProviderItem;
|
|
316
|
+
boundaryId: string;
|
|
317
|
+
getFile: (fileId: string) => Promise<GoogleDriveInventoryProviderItem | null>;
|
|
318
|
+
}): Promise<"inside" | "outside" | "unknown"> {
|
|
319
|
+
if (input.file.id === input.boundaryId) return "inside";
|
|
320
|
+
let parents = input.file.parents;
|
|
321
|
+
const seen = new Set<string>([input.file.id]);
|
|
322
|
+
for (let depth = 0; depth < MAX_ANCESTOR_DEPTH; depth += 1) {
|
|
323
|
+
if (parents.includes(input.boundaryId)) return "inside";
|
|
324
|
+
const next = parents.find((parent) => !seen.has(parent));
|
|
325
|
+
if (!next) return "outside";
|
|
326
|
+
seen.add(next);
|
|
327
|
+
const parent = await input.getFile(next);
|
|
328
|
+
if (!parent) return "unknown";
|
|
329
|
+
parents = parent.parents;
|
|
330
|
+
}
|
|
331
|
+
return "unknown";
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function toInventoryEntry(
|
|
335
|
+
file: GoogleDriveInventoryProviderItem,
|
|
336
|
+
source: GoogleDriveSelectedSource,
|
|
337
|
+
maxFileBytes: number,
|
|
338
|
+
): GoogleDriveInventoryEntry {
|
|
339
|
+
return {
|
|
340
|
+
externalObjectId: file.id,
|
|
341
|
+
externalVersionId: file.version ?? file.modifiedTime ?? file.md5Checksum,
|
|
342
|
+
sourceId: source.id,
|
|
343
|
+
parentFolderId: file.parents[0] ?? source.id,
|
|
344
|
+
driveId: file.driveId ?? source.driveId,
|
|
345
|
+
title: file.name,
|
|
346
|
+
mimeType: file.mimeType,
|
|
347
|
+
modifiedTime: file.modifiedTime,
|
|
348
|
+
createdTime: file.createdTime,
|
|
349
|
+
sourceUri:
|
|
350
|
+
file.webViewLink ?? `https://drive.google.com/open?id=${encodeURIComponent(file.id)}`,
|
|
351
|
+
transfer: planGoogleDriveTransfer(file, maxFileBytes),
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function validatedCheckpoint(
|
|
356
|
+
value: unknown,
|
|
357
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
358
|
+
): GoogleDriveChangesCheckpoint {
|
|
359
|
+
if (!value || typeof value !== "object") {
|
|
360
|
+
throw new Error("google_drive_changes_checkpoint_invalid");
|
|
361
|
+
}
|
|
362
|
+
const checkpoint = value as Record<string, unknown>;
|
|
363
|
+
const budget = checkpoint.budget;
|
|
364
|
+
if (
|
|
365
|
+
checkpoint.version !== 2 ||
|
|
366
|
+
typeof checkpoint.requiresFullReconciliation !== "boolean" ||
|
|
367
|
+
!Array.isArray(checkpoint.seenPageTokenHashes) ||
|
|
368
|
+
checkpoint.seenPageTokenHashes.some((hash) => !/^[a-f0-9]{64}$/.test(hash)) ||
|
|
369
|
+
new Set(checkpoint.seenPageTokenHashes).size !== checkpoint.seenPageTokenHashes.length ||
|
|
370
|
+
checkpoint.seenPageTokenHashes.length > limits.maxProviderRequests ||
|
|
371
|
+
!budget ||
|
|
372
|
+
typeof budget !== "object"
|
|
373
|
+
) {
|
|
374
|
+
throw new Error("google_drive_changes_checkpoint_invalid");
|
|
375
|
+
}
|
|
376
|
+
if (typeof checkpoint.pageToken !== "string") {
|
|
377
|
+
throw new Error("google_drive_changes_checkpoint_invalid");
|
|
378
|
+
}
|
|
379
|
+
boundedToken(checkpoint.pageToken);
|
|
380
|
+
const budgetRow = budget as Record<string, unknown>;
|
|
381
|
+
for (const [key, maximum] of [
|
|
382
|
+
["examinedItems", limits.maxItems],
|
|
383
|
+
["providerRequests", limits.maxProviderRequests],
|
|
384
|
+
["elapsedMs", limits.maxElapsedMs],
|
|
385
|
+
] as const) {
|
|
386
|
+
const budgetValue = budgetRow[key];
|
|
387
|
+
if (
|
|
388
|
+
!Number.isSafeInteger(budgetValue) ||
|
|
389
|
+
Number(budgetValue) < 0 ||
|
|
390
|
+
Number(budgetValue) > maximum
|
|
391
|
+
) {
|
|
392
|
+
throw new Error("google_drive_changes_checkpoint_budget_invalid");
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
if (checkpoint.seenPageTokenHashes.length > Number(budgetRow.providerRequests)) {
|
|
396
|
+
throw new Error("google_drive_changes_checkpoint_request_history_invalid");
|
|
397
|
+
}
|
|
398
|
+
return value as GoogleDriveChangesCheckpoint;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function limitReason(
|
|
402
|
+
budget: GoogleDriveSyncBudget,
|
|
403
|
+
input: {
|
|
404
|
+
maxItems: number;
|
|
405
|
+
maxProviderRequests: number;
|
|
406
|
+
maxElapsedMs: number;
|
|
407
|
+
maxInvocationElapsedMs: number;
|
|
408
|
+
},
|
|
409
|
+
invocationElapsedMs: number,
|
|
410
|
+
): GoogleDriveChangesStopReason | null {
|
|
411
|
+
if (budget.examinedItems >= input.maxItems) return "item_limit";
|
|
412
|
+
if (budget.providerRequests >= input.maxProviderRequests) return "api_request_limit";
|
|
413
|
+
if (
|
|
414
|
+
budget.elapsedMs >= input.maxElapsedMs ||
|
|
415
|
+
invocationElapsedMs >= input.maxInvocationElapsedMs
|
|
416
|
+
) {
|
|
417
|
+
return "elapsed_time_limit";
|
|
418
|
+
}
|
|
419
|
+
return null;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function emptyBudget(): GoogleDriveSyncBudget {
|
|
423
|
+
return { examinedItems: 0, providerRequests: 0, elapsedMs: 0 };
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function boundedToken(value: string): string {
|
|
427
|
+
if (value.length < 1 || value.length > MAX_TOKEN_CHARS) {
|
|
428
|
+
throw new Error("google_drive_changes_token_invalid");
|
|
429
|
+
}
|
|
430
|
+
return value;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function boundedBoundaryId(value: string): string {
|
|
434
|
+
if (value.length < 1 || value.length > MAX_BOUNDARY_ID_CHARS) {
|
|
435
|
+
throw new Error("google_drive_changes_boundary_invalid");
|
|
436
|
+
}
|
|
437
|
+
return value;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
export class GoogleDriveCursorInvalidError extends Error {
|
|
441
|
+
budget: GoogleDriveSyncBudget | null = null;
|
|
442
|
+
|
|
443
|
+
constructor() {
|
|
444
|
+
super("google_drive_changes_cursor_invalid");
|
|
445
|
+
this.name = "GoogleDriveCursorInvalidError";
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
export class GoogleDriveChangesProtocolError extends Error {
|
|
450
|
+
constructor(message: string) {
|
|
451
|
+
super(message);
|
|
452
|
+
this.name = "GoogleDriveChangesProtocolError";
|
|
453
|
+
}
|
|
454
|
+
}
|