@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,2840 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { heartbeat } from "@temporalio/activity";
|
|
3
|
+
import type { Settings } from "@opengeni/config";
|
|
4
|
+
import {
|
|
5
|
+
KnowledgeSourceSyncRunSummary,
|
|
6
|
+
type ScheduledTask,
|
|
7
|
+
type ScopedKnowledgeScope,
|
|
8
|
+
} from "@opengeni/contracts";
|
|
9
|
+
import {
|
|
10
|
+
ATLASSIAN_PROVIDER_DOMAIN,
|
|
11
|
+
AtlassianConnectionMetadata,
|
|
12
|
+
atlassianScopesAllowRead,
|
|
13
|
+
type AtlassianSelectedSource,
|
|
14
|
+
} from "@opengeni/contracts/atlassian";
|
|
15
|
+
import {
|
|
16
|
+
GoogleDriveConnectionMetadata,
|
|
17
|
+
GOOGLE_DRIVE_PROVIDER_DOMAIN,
|
|
18
|
+
googleDriveScopesAllowCapability,
|
|
19
|
+
type GoogleDriveSelectedSource,
|
|
20
|
+
} from "@opengeni/contracts/google-drive";
|
|
21
|
+
import {
|
|
22
|
+
googleDriveKnowledgeScope,
|
|
23
|
+
inventoryGoogleDriveSource,
|
|
24
|
+
type GoogleDriveInventoryEntry,
|
|
25
|
+
type GoogleDriveInventoryCheckpoint,
|
|
26
|
+
type GoogleDriveInventoryPage,
|
|
27
|
+
type GoogleDriveInventoryProviderItem,
|
|
28
|
+
type GoogleDriveInventoryStopReason,
|
|
29
|
+
} from "@opengeni/documents/google-drive";
|
|
30
|
+
import {
|
|
31
|
+
atlassianKnowledgeScope,
|
|
32
|
+
inventoryAtlassianSource,
|
|
33
|
+
type AtlassianInventoryEntry,
|
|
34
|
+
type AtlassianInventoryPage,
|
|
35
|
+
type AtlassianInventoryStopReason,
|
|
36
|
+
} from "@opengeni/documents/atlassian";
|
|
37
|
+
import { addDocumentToBase, ensureDefaultBase, getDocumentBase } from "@opengeni/documents";
|
|
38
|
+
import {
|
|
39
|
+
appendKnowledgeDocumentVersion,
|
|
40
|
+
appendKnowledgeSourceAclVersion,
|
|
41
|
+
beginKnowledgeSyncRun,
|
|
42
|
+
buildConnectionTokenResolver,
|
|
43
|
+
checkpointKnowledgeSourceSync,
|
|
44
|
+
claimKnowledgeSourceSyncLease,
|
|
45
|
+
compareCanonicalDecimalProviderRevisions,
|
|
46
|
+
completeKnowledgeSyncRun,
|
|
47
|
+
completeKnowledgeSourceSyncWakeWithoutLease,
|
|
48
|
+
deauthorizeKnowledgeSourceRetrieval,
|
|
49
|
+
enqueueKnowledgeSourceSyncIndexObligation,
|
|
50
|
+
ensureKnowledgeSourceBlobFile,
|
|
51
|
+
getConnectionMetadata,
|
|
52
|
+
getKnowledgeDocumentVersionObservationForSyncAuthority,
|
|
53
|
+
getKnowledgeSyncRunForSyncAuthority,
|
|
54
|
+
getKnowledgeSourceAclForSyncAuthority,
|
|
55
|
+
getKnowledgeSourceForSyncAuthority,
|
|
56
|
+
getKnowledgeSourceObjectForSyncAuthority,
|
|
57
|
+
getKnowledgeSourceSyncIndexObligationForVersion,
|
|
58
|
+
getScheduledTask,
|
|
59
|
+
listObservedKnowledgeSourceSyncExternalObjectIds,
|
|
60
|
+
KnowledgeSourceSyncObservationFenceError,
|
|
61
|
+
reconcileKnowledgeSourceSyncCompleteScan,
|
|
62
|
+
reconcileKnowledgeSourceSyncLiveGeneration,
|
|
63
|
+
recordKnowledgeSourceSyncItemOutcomes,
|
|
64
|
+
recordKnowledgeSourceSyncObjectObservations,
|
|
65
|
+
recordUsageEvent,
|
|
66
|
+
releaseKnowledgeSourceSyncLeaseForRetry,
|
|
67
|
+
restoreKnowledgeSourceObject,
|
|
68
|
+
retryKnowledgeSourceSyncIndexObligation,
|
|
69
|
+
scopedKnowledgeScopeKey,
|
|
70
|
+
settleKnowledgeSourceSyncLease,
|
|
71
|
+
settleKnowledgeSourceSyncIndexObligation,
|
|
72
|
+
updateKnowledgeSourceDocumentObservationMetadata,
|
|
73
|
+
updateScheduledTaskRun,
|
|
74
|
+
upsertKnowledgeSourceObject,
|
|
75
|
+
type Database,
|
|
76
|
+
type KnowledgeSourceSyncObjectObservationResult,
|
|
77
|
+
type KnowledgeSourceSyncObservationFloor,
|
|
78
|
+
} from "@opengeni/db";
|
|
79
|
+
import { readResponseJsonBounded } from "@opengeni/network";
|
|
80
|
+
import { createDocumentActivities } from "./documents";
|
|
81
|
+
import type {
|
|
82
|
+
ControlActivityServices,
|
|
83
|
+
RunKnowledgeSourceSyncBatchInput,
|
|
84
|
+
RunKnowledgeSourceSyncBatchResult,
|
|
85
|
+
} from "./types";
|
|
86
|
+
import type { KnowledgeSourceSyncDriver } from "./knowledge-source-sync-driver";
|
|
87
|
+
import {
|
|
88
|
+
advanceGoogleDriveChangesCursor,
|
|
89
|
+
buildGoogleDriveChangesCursor,
|
|
90
|
+
drainGoogleDriveChanges,
|
|
91
|
+
GoogleDriveChangesProtocolError,
|
|
92
|
+
GoogleDriveCursorInvalidError,
|
|
93
|
+
googleDriveFullReconciliationDue,
|
|
94
|
+
parseGoogleDriveChangesCheckpoint,
|
|
95
|
+
parseGoogleDriveChangesCursor,
|
|
96
|
+
resolveGoogleDriveChangesBoundaryId,
|
|
97
|
+
type GoogleDriveChangesCheckpoint,
|
|
98
|
+
type GoogleDriveChangesPage,
|
|
99
|
+
type GoogleDriveSyncBudget,
|
|
100
|
+
} from "./google-drive-changes";
|
|
101
|
+
|
|
102
|
+
const DRIVE_API_BASE = "https://www.googleapis.com/drive/v3";
|
|
103
|
+
const DRIVE_JSON_MAX_BYTES = 2 * 1024 * 1024;
|
|
104
|
+
const GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
|
105
|
+
const GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS = 30_000;
|
|
106
|
+
const GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES = 2 * 1024 * 1024;
|
|
107
|
+
const ATLASSIAN_API_BASE = "https://api.atlassian.com";
|
|
108
|
+
const ATLASSIAN_JSON_MAX_BYTES = 4 * 1024 * 1024;
|
|
109
|
+
|
|
110
|
+
type KnowledgeSyncEntry = GoogleDriveInventoryEntry | AtlassianInventoryEntry;
|
|
111
|
+
type KnowledgeSyncStopReason = GoogleDriveInventoryStopReason | AtlassianInventoryStopReason;
|
|
112
|
+
|
|
113
|
+
export function knowledgeSyncObservationPolicy(kind: "google_drive" | "atlassian") {
|
|
114
|
+
return kind === "google_drive"
|
|
115
|
+
? { revisionOrdering: "canonical_decimal" as const, filterWithDriveDurability: true }
|
|
116
|
+
: { revisionOrdering: "first_observation" as const, filterWithDriveDurability: false };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function createKnowledgeSourceSyncActivities(
|
|
120
|
+
services: () => Promise<ControlActivityServices>,
|
|
121
|
+
resolveDocumentServices?: () => Promise<import("@opengeni/documents").DocumentServices>,
|
|
122
|
+
) {
|
|
123
|
+
const documentActivities = createDocumentActivities(services, resolveDocumentServices);
|
|
124
|
+
return {
|
|
125
|
+
runKnowledgeSourceSyncBatch: async (
|
|
126
|
+
input: RunKnowledgeSourceSyncBatchInput,
|
|
127
|
+
): Promise<RunKnowledgeSourceSyncBatchResult> => {
|
|
128
|
+
const { settings, db, objectStorage, observability } = await services();
|
|
129
|
+
if (!objectStorage) throw new Error("object storage is not configured");
|
|
130
|
+
const task = await getScheduledTask(db, input.workspaceId, input.taskId);
|
|
131
|
+
if (
|
|
132
|
+
!task ||
|
|
133
|
+
task.accountId !== input.accountId ||
|
|
134
|
+
task.action.kind !== "knowledge_source_sync" ||
|
|
135
|
+
task.action.sourceId !== input.sourceId ||
|
|
136
|
+
task.status !== "active"
|
|
137
|
+
) {
|
|
138
|
+
return await failWithoutProvider("schedule_inactive_or_changed");
|
|
139
|
+
}
|
|
140
|
+
const action = task.action;
|
|
141
|
+
const providerLabel =
|
|
142
|
+
action.connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN
|
|
143
|
+
? "atlassian"
|
|
144
|
+
: "google_drive";
|
|
145
|
+
if (
|
|
146
|
+
action.controlWorkspaceId !== input.workspaceId ||
|
|
147
|
+
action.sourceConfigGeneration <= 0 ||
|
|
148
|
+
!action.providerCoordinationKey
|
|
149
|
+
) {
|
|
150
|
+
return await failWithoutProvider("authority_changed");
|
|
151
|
+
}
|
|
152
|
+
const lease = await claimKnowledgeSourceSyncLease(db, {
|
|
153
|
+
...input,
|
|
154
|
+
overlapPolicy: input.overlapPolicy,
|
|
155
|
+
});
|
|
156
|
+
if (lease.action !== "claimed") {
|
|
157
|
+
observability.incrementCounter({
|
|
158
|
+
name: "opengeni_knowledge_source_sync_runs_total",
|
|
159
|
+
labels: { provider: providerLabel, outcome: lease.action },
|
|
160
|
+
});
|
|
161
|
+
return { action: lease.action };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
let knowledgeRunId: string | null = null;
|
|
165
|
+
let activeSourceSyncGeneration: number | null = null;
|
|
166
|
+
const startedAt = Date.now();
|
|
167
|
+
const summary: KnowledgeSourceSyncRunSummary = {
|
|
168
|
+
phase: "queued",
|
|
169
|
+
scanned: 0,
|
|
170
|
+
imported: 0,
|
|
171
|
+
unchanged: 0,
|
|
172
|
+
skipped: 0,
|
|
173
|
+
failed: 0,
|
|
174
|
+
bytes: 0,
|
|
175
|
+
providerRequests: 0,
|
|
176
|
+
elapsedMs: 0,
|
|
177
|
+
indexed: 0,
|
|
178
|
+
aclPending: 0,
|
|
179
|
+
retryable: false,
|
|
180
|
+
limitReached: null as
|
|
181
|
+
| "items"
|
|
182
|
+
| "bytes"
|
|
183
|
+
| "file_bytes"
|
|
184
|
+
| "provider_requests"
|
|
185
|
+
| "elapsed_time"
|
|
186
|
+
| null,
|
|
187
|
+
checkpointed: false,
|
|
188
|
+
reconnectRequired: false,
|
|
189
|
+
failures: [],
|
|
190
|
+
};
|
|
191
|
+
try {
|
|
192
|
+
summary.phase = "inventory";
|
|
193
|
+
const resolved = await getKnowledgeSourceForSyncAuthority(db, {
|
|
194
|
+
accountId: input.accountId,
|
|
195
|
+
workspaceId: input.workspaceId,
|
|
196
|
+
sourceId: input.sourceId,
|
|
197
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
198
|
+
});
|
|
199
|
+
if (!resolved || resolved.source.lifecycleState !== "active") {
|
|
200
|
+
throw new SyncFailure("authority_changed", false);
|
|
201
|
+
}
|
|
202
|
+
if (
|
|
203
|
+
scopedKnowledgeScopeKey(resolved.source.scope) !==
|
|
204
|
+
scopedKnowledgeScopeKey(action.destination)
|
|
205
|
+
) {
|
|
206
|
+
throw new SyncFailure("authority_changed", false);
|
|
207
|
+
}
|
|
208
|
+
const liveState = await reconcileKnowledgeSourceSyncLiveGeneration(db, {
|
|
209
|
+
accountId: input.accountId,
|
|
210
|
+
workspaceId: input.workspaceId,
|
|
211
|
+
sourceId: input.sourceId,
|
|
212
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
213
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
214
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
215
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
216
|
+
liveSourceSyncGeneration: resolved.source.syncGeneration,
|
|
217
|
+
});
|
|
218
|
+
activeSourceSyncGeneration = resolved.source.syncGeneration;
|
|
219
|
+
const syncOperationId = `scheduled-task-run:${input.scheduledTaskRunId}`;
|
|
220
|
+
const replayedRun = await getKnowledgeSyncRunForSyncAuthority(db, {
|
|
221
|
+
accountId: input.accountId,
|
|
222
|
+
workspaceId: input.workspaceId,
|
|
223
|
+
sourceId: input.sourceId,
|
|
224
|
+
operationId: syncOperationId,
|
|
225
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
226
|
+
});
|
|
227
|
+
if (replayedRun && replayedRun.state !== "started") {
|
|
228
|
+
const replayedSummary = KnowledgeSourceSyncRunSummary.parse(replayedRun.metadata);
|
|
229
|
+
const settlementInput: Parameters<typeof settleKnowledgeSourceSyncLease>[1] = {
|
|
230
|
+
accountId: input.accountId,
|
|
231
|
+
workspaceId: input.workspaceId,
|
|
232
|
+
sourceId: input.sourceId,
|
|
233
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
234
|
+
knowledgeSyncRunId: replayedRun.id,
|
|
235
|
+
status: replayedRun.state === "succeeded" ? "succeeded" : "failed",
|
|
236
|
+
summary: replayedSummary,
|
|
237
|
+
error: replayedRun.errorCode,
|
|
238
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
239
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
240
|
+
sourceSyncGeneration: replayedRun.inputSyncGeneration,
|
|
241
|
+
completedSourceSyncGeneration:
|
|
242
|
+
replayedRun.state === "succeeded" ? replayedRun.inputSyncGeneration + 1 : null,
|
|
243
|
+
...(replayedRun.state === "succeeded" &&
|
|
244
|
+
replayedRun.metadata.providerCursor &&
|
|
245
|
+
typeof replayedRun.metadata.providerCursor === "object"
|
|
246
|
+
? {
|
|
247
|
+
providerCursor: replayedRun.metadata.providerCursor as Record<string, unknown>,
|
|
248
|
+
}
|
|
249
|
+
: {}),
|
|
250
|
+
};
|
|
251
|
+
const settled =
|
|
252
|
+
replayedRun.state === "failed" && replayedSummary.reconnectRequired
|
|
253
|
+
? await deauthorizeAndSettleKnowledgeSourceSyncFailure(db, {
|
|
254
|
+
deauthorization: {
|
|
255
|
+
accountId: input.accountId,
|
|
256
|
+
workspaceId: input.workspaceId,
|
|
257
|
+
sourceId: input.sourceId,
|
|
258
|
+
audience: action.destination,
|
|
259
|
+
operationId: `knowledge-sync-deauthorize:${input.sourceId}:${action.connection.connectionVersion}:${replayedRun.errorCode ?? "connection_reconnect_required"}`,
|
|
260
|
+
reasonCode: "connection_reconnect_required",
|
|
261
|
+
actor: {
|
|
262
|
+
kind: "service",
|
|
263
|
+
subjectId: "knowledge-source-sync",
|
|
264
|
+
initiatingHumanSubjectId: action.initiatingSubjectId,
|
|
265
|
+
},
|
|
266
|
+
},
|
|
267
|
+
settlement: settlementInput,
|
|
268
|
+
})
|
|
269
|
+
: await settleKnowledgeSourceSyncLease(db, settlementInput);
|
|
270
|
+
await recordConnectorUsage("completed", replayedSummary.scanned, "item");
|
|
271
|
+
await recordConnectorUsage("items", replayedSummary.imported, "item");
|
|
272
|
+
await recordConnectorUsage("bytes", replayedSummary.bytes, "byte");
|
|
273
|
+
return {
|
|
274
|
+
action: replayedRun.state === "succeeded" ? "complete" : "failed",
|
|
275
|
+
bufferedWake: settled.bufferedWake,
|
|
276
|
+
bufferedScheduledTaskRunId: settled.bufferedScheduledTaskRunId,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
const provider = await resolveKnowledgeSyncProvider({
|
|
280
|
+
db,
|
|
281
|
+
settings,
|
|
282
|
+
accountId: input.accountId,
|
|
283
|
+
workspaceId: input.workspaceId,
|
|
284
|
+
action,
|
|
285
|
+
sourceId: input.sourceId,
|
|
286
|
+
externalSourceId: resolved.source.externalSourceId,
|
|
287
|
+
});
|
|
288
|
+
const selectedDestination = provider.selectedDestination;
|
|
289
|
+
const observationPolicy = knowledgeSyncObservationPolicy(provider.kind);
|
|
290
|
+
|
|
291
|
+
let aclGeneration = resolved.source.currentAclGeneration;
|
|
292
|
+
let acl = aclGeneration
|
|
293
|
+
? await getKnowledgeSourceAclForSyncAuthority(db, {
|
|
294
|
+
accountId: input.accountId,
|
|
295
|
+
workspaceId: input.workspaceId,
|
|
296
|
+
sourceId: input.sourceId,
|
|
297
|
+
generation: aclGeneration,
|
|
298
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
299
|
+
})
|
|
300
|
+
: null;
|
|
301
|
+
const actor = {
|
|
302
|
+
kind: "service" as const,
|
|
303
|
+
subjectId: "knowledge-source-sync",
|
|
304
|
+
initiatingHumanSubjectId: action.initiatingSubjectId,
|
|
305
|
+
};
|
|
306
|
+
if (!acl) {
|
|
307
|
+
acl = await appendKnowledgeSourceAclVersion(db, {
|
|
308
|
+
accountId: input.accountId,
|
|
309
|
+
workspaceId: input.workspaceId,
|
|
310
|
+
sourceId: input.sourceId,
|
|
311
|
+
audience: action.destination,
|
|
312
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
313
|
+
expectedAclGeneration: 0,
|
|
314
|
+
aclVersion: "opengeni-destination-v1",
|
|
315
|
+
agentAccess: false,
|
|
316
|
+
operationId: `source-initial-acl:${input.sourceId}`,
|
|
317
|
+
reasonCode: "initial_destination_acl",
|
|
318
|
+
actor,
|
|
319
|
+
});
|
|
320
|
+
aclGeneration = acl.generation;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const knowledgeRun =
|
|
324
|
+
replayedRun ??
|
|
325
|
+
(await beginKnowledgeSyncRun(db, {
|
|
326
|
+
accountId: input.accountId,
|
|
327
|
+
workspaceId: input.workspaceId,
|
|
328
|
+
sourceId: input.sourceId,
|
|
329
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
330
|
+
expectedSyncGeneration: resolved.source.syncGeneration,
|
|
331
|
+
inputCursor: null,
|
|
332
|
+
operationId: syncOperationId,
|
|
333
|
+
actor,
|
|
334
|
+
}));
|
|
335
|
+
knowledgeRunId = knowledgeRun.id;
|
|
336
|
+
|
|
337
|
+
const driver = provider.driver;
|
|
338
|
+
const inventory = await driver.inventory(
|
|
339
|
+
lease.state.executionCheckpoint,
|
|
340
|
+
lease.state.providerCursor,
|
|
341
|
+
);
|
|
342
|
+
summary.providerRequests = inventory.providerRequests;
|
|
343
|
+
summary.elapsedMs = inventory.elapsedMs;
|
|
344
|
+
|
|
345
|
+
const base = selectedDestination.collectionId
|
|
346
|
+
? await getDocumentBase(db, input.workspaceId, selectedDestination.collectionId)
|
|
347
|
+
: await ensureDefaultBase(db, {
|
|
348
|
+
accountId: input.accountId,
|
|
349
|
+
workspaceId: input.workspaceId,
|
|
350
|
+
});
|
|
351
|
+
if (!base) throw new SyncFailure("authority_changed", false);
|
|
352
|
+
|
|
353
|
+
const durableObservations = await recordKnowledgeSourceSyncObjectObservations(db, {
|
|
354
|
+
accountId: input.accountId,
|
|
355
|
+
workspaceId: input.workspaceId,
|
|
356
|
+
sourceId: input.sourceId,
|
|
357
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
358
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
359
|
+
scanGeneration: liveState.activeScanGeneration,
|
|
360
|
+
executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
|
|
361
|
+
revisionOrdering: observationPolicy.revisionOrdering,
|
|
362
|
+
observations: inventory.entries.map((entry) => ({
|
|
363
|
+
externalObjectId: entry.externalObjectId,
|
|
364
|
+
providerRevision: entry.externalVersionId,
|
|
365
|
+
metadataHash: entryMetadataHash(entry),
|
|
366
|
+
})),
|
|
367
|
+
});
|
|
368
|
+
const entriesToProcess = observationPolicy.filterWithDriveDurability
|
|
369
|
+
? inventory.entries.filter((entry, index) =>
|
|
370
|
+
shouldProcessGoogleDriveDurableObservation({
|
|
371
|
+
entry,
|
|
372
|
+
observation: durableObservations[index]!,
|
|
373
|
+
sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
374
|
+
aclGeneration: aclGeneration!,
|
|
375
|
+
}),
|
|
376
|
+
)
|
|
377
|
+
: inventory.entries;
|
|
378
|
+
const observationFloorsById = new Map<string, KnowledgeSourceSyncObservationFloor>();
|
|
379
|
+
for (const observation of durableObservations) {
|
|
380
|
+
observationFloorsById.set(observation.externalObjectId, {
|
|
381
|
+
externalObjectId: observation.externalObjectId,
|
|
382
|
+
providerRevision: observation.providerRevision,
|
|
383
|
+
metadataHash: observation.metadataHash,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
const observationFloors = [...observationFloorsById.values()];
|
|
387
|
+
const executionCheckpoint = mergeGoogleDriveDurableObservationFloors(
|
|
388
|
+
inventory.checkpoint,
|
|
389
|
+
observationFloors,
|
|
390
|
+
action.limits.maxItems,
|
|
391
|
+
);
|
|
392
|
+
|
|
393
|
+
const ensureVersionIndexed = async (details: {
|
|
394
|
+
entry: KnowledgeSyncEntry;
|
|
395
|
+
object: {
|
|
396
|
+
id: string;
|
|
397
|
+
lifecycleGeneration: number;
|
|
398
|
+
};
|
|
399
|
+
version: {
|
|
400
|
+
id: string;
|
|
401
|
+
versionGeneration: number;
|
|
402
|
+
documentId: string | null;
|
|
403
|
+
};
|
|
404
|
+
indexRequired: boolean;
|
|
405
|
+
obligation?: Awaited<ReturnType<typeof getKnowledgeSourceSyncIndexObligationForVersion>>;
|
|
406
|
+
}) => {
|
|
407
|
+
if (!details.version.documentId) {
|
|
408
|
+
throw new SyncFailure("authority_changed", false);
|
|
409
|
+
}
|
|
410
|
+
let obligation =
|
|
411
|
+
details.obligation ??
|
|
412
|
+
(await getKnowledgeSourceSyncIndexObligationForVersion(db, {
|
|
413
|
+
accountId: input.accountId,
|
|
414
|
+
workspaceId: input.workspaceId,
|
|
415
|
+
knowledgeDocumentVersionId: details.version.id,
|
|
416
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
417
|
+
}));
|
|
418
|
+
if (!obligation) {
|
|
419
|
+
obligation = await enqueueKnowledgeSourceSyncIndexObligation(db, {
|
|
420
|
+
accountId: input.accountId,
|
|
421
|
+
workspaceId: input.workspaceId,
|
|
422
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
423
|
+
sourceId: input.sourceId,
|
|
424
|
+
sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
|
|
425
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
426
|
+
externalObjectId: details.entry.externalObjectId,
|
|
427
|
+
knowledgeSourceObjectId: details.object.id,
|
|
428
|
+
knowledgeDocumentVersionId: details.version.id,
|
|
429
|
+
documentId: details.version.documentId,
|
|
430
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
431
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
432
|
+
objectLifecycleGeneration: details.object.lifecycleGeneration,
|
|
433
|
+
objectVersionGeneration: details.version.versionGeneration,
|
|
434
|
+
citationLocator: driver.citationLocator(details.entry),
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
if (obligation.documentId !== details.version.documentId) {
|
|
438
|
+
throw new SyncFailure("authority_changed", false);
|
|
439
|
+
}
|
|
440
|
+
if (obligation.status === "indexed") return obligation;
|
|
441
|
+
if (obligation.status === "failed") {
|
|
442
|
+
const retried = await retryKnowledgeSourceSyncIndexObligation(db, {
|
|
443
|
+
accountId: input.accountId,
|
|
444
|
+
workspaceId: input.workspaceId,
|
|
445
|
+
obligationId: obligation.id,
|
|
446
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
447
|
+
});
|
|
448
|
+
if (retried === "invalidated") {
|
|
449
|
+
throw new SyncFailure("authority_changed", false);
|
|
450
|
+
}
|
|
451
|
+
obligation = { ...obligation, status: retried };
|
|
452
|
+
if (retried === "indexed") return obligation;
|
|
453
|
+
}
|
|
454
|
+
if (obligation.status !== "pending") {
|
|
455
|
+
throw new SyncFailure("authority_changed", false);
|
|
456
|
+
}
|
|
457
|
+
if (details.indexRequired) {
|
|
458
|
+
summary.phase = "index";
|
|
459
|
+
try {
|
|
460
|
+
const indexedDocument = await documentActivities.indexDocument({
|
|
461
|
+
accountId: input.accountId,
|
|
462
|
+
workspaceId: input.workspaceId,
|
|
463
|
+
documentId: details.version.documentId,
|
|
464
|
+
authorityKind: action.destination.kind,
|
|
465
|
+
authorityWorkspaceId: action.destination.workspaceId,
|
|
466
|
+
authoritySubjectId: action.destination.subjectId,
|
|
467
|
+
});
|
|
468
|
+
if (indexedDocument.status !== "ready") {
|
|
469
|
+
throw new Error("document indexing did not complete");
|
|
470
|
+
}
|
|
471
|
+
} catch {
|
|
472
|
+
await settleKnowledgeSourceSyncIndexObligation(db, {
|
|
473
|
+
accountId: input.accountId,
|
|
474
|
+
workspaceId: input.workspaceId,
|
|
475
|
+
obligationId: obligation.id,
|
|
476
|
+
status: "failed",
|
|
477
|
+
failureCode: "indexing_failed",
|
|
478
|
+
}).catch(() => undefined);
|
|
479
|
+
throw new SyncFailure("indexing_failed", true);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
const settlement = await settleKnowledgeSourceSyncIndexObligation(db, {
|
|
483
|
+
accountId: input.accountId,
|
|
484
|
+
workspaceId: input.workspaceId,
|
|
485
|
+
obligationId: obligation.id,
|
|
486
|
+
status: "indexed",
|
|
487
|
+
});
|
|
488
|
+
if (settlement === "invalidated") {
|
|
489
|
+
throw new SyncFailure("authority_changed", false);
|
|
490
|
+
}
|
|
491
|
+
return { ...obligation, status: "indexed" };
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
const outcomes = [];
|
|
495
|
+
for (const entry of entriesToProcess) {
|
|
496
|
+
heartbeat({ sourceId: input.sourceId, externalObjectId: entry.externalObjectId });
|
|
497
|
+
const observationFloor = observationFloorsById.get(entry.externalObjectId);
|
|
498
|
+
if (!observationFloor) throw new SyncFailure("provider_payload_invalid", false);
|
|
499
|
+
const syncObservation = {
|
|
500
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
501
|
+
scanGeneration: liveState.activeScanGeneration,
|
|
502
|
+
providerRevision: observationFloor.providerRevision,
|
|
503
|
+
metadataHash: observationFloor.metadataHash,
|
|
504
|
+
};
|
|
505
|
+
summary.scanned += 1;
|
|
506
|
+
if (entry.transfer.action === "skip") {
|
|
507
|
+
summary.skipped += 1;
|
|
508
|
+
outcomes.push({
|
|
509
|
+
externalObjectId: entry.externalObjectId,
|
|
510
|
+
outcome: "skipped" as const,
|
|
511
|
+
reasonCode: entry.transfer.reason,
|
|
512
|
+
});
|
|
513
|
+
continue;
|
|
514
|
+
}
|
|
515
|
+
if (entry.transfer.action === "traverse") continue;
|
|
516
|
+
try {
|
|
517
|
+
const existingObject = await getKnowledgeSourceObjectForSyncAuthority(db, {
|
|
518
|
+
accountId: input.accountId,
|
|
519
|
+
workspaceId: input.workspaceId,
|
|
520
|
+
sourceId: input.sourceId,
|
|
521
|
+
externalObjectId: entry.externalObjectId,
|
|
522
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
523
|
+
});
|
|
524
|
+
let object =
|
|
525
|
+
existingObject ??
|
|
526
|
+
(await upsertKnowledgeSourceObject(db, {
|
|
527
|
+
accountId: input.accountId,
|
|
528
|
+
workspaceId: input.workspaceId,
|
|
529
|
+
sourceId: input.sourceId,
|
|
530
|
+
externalObjectId: entry.externalObjectId,
|
|
531
|
+
operationId: `source-object:${input.sourceId}:${entry.externalObjectId}`,
|
|
532
|
+
actor,
|
|
533
|
+
}));
|
|
534
|
+
if (object.lifecycleState !== "active") {
|
|
535
|
+
const restored = await restoreKnowledgeSourceObject(db, {
|
|
536
|
+
accountId: input.accountId,
|
|
537
|
+
workspaceId: input.workspaceId,
|
|
538
|
+
targetId: object.id,
|
|
539
|
+
expectedGeneration: object.lifecycleGeneration,
|
|
540
|
+
operationId: `knowledge-sync-object-restore:${object.id}:${liveState.activeScanGeneration}`,
|
|
541
|
+
reasonCode: "authoritative_scan_observed",
|
|
542
|
+
actor,
|
|
543
|
+
});
|
|
544
|
+
object = {
|
|
545
|
+
...object,
|
|
546
|
+
lifecycleState: "active",
|
|
547
|
+
lifecycleGeneration: restored.lifecycleGeneration,
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
const currentObservation = object.currentVersionId
|
|
551
|
+
? await getKnowledgeDocumentVersionObservationForSyncAuthority(db, {
|
|
552
|
+
accountId: input.accountId,
|
|
553
|
+
workspaceId: input.workspaceId,
|
|
554
|
+
versionId: object.currentVersionId,
|
|
555
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
556
|
+
})
|
|
557
|
+
: null;
|
|
558
|
+
const currentVersion = currentObservation?.version ?? null;
|
|
559
|
+
const metadataHash = entryMetadataHash(entry);
|
|
560
|
+
const observedProviderRevision =
|
|
561
|
+
typeof currentObservation?.sourceMetadata.providerRevision === "string"
|
|
562
|
+
? currentObservation.sourceMetadata.providerRevision
|
|
563
|
+
: null;
|
|
564
|
+
const observedMetadataHash =
|
|
565
|
+
typeof currentObservation?.sourceMetadata.metadataHash === "string"
|
|
566
|
+
? currentObservation.sourceMetadata.metadataHash
|
|
567
|
+
: null;
|
|
568
|
+
const observedSourceLifecycleGeneration = Number(
|
|
569
|
+
currentObservation?.sourceMetadata.sourceLifecycleGeneration ?? 0,
|
|
570
|
+
);
|
|
571
|
+
const observedObjectLifecycleGeneration = Number(
|
|
572
|
+
currentObservation?.sourceMetadata.objectLifecycleGeneration ?? 0,
|
|
573
|
+
);
|
|
574
|
+
const currentObligation = currentVersion
|
|
575
|
+
? await getKnowledgeSourceSyncIndexObligationForVersion(db, {
|
|
576
|
+
accountId: input.accountId,
|
|
577
|
+
workspaceId: input.workspaceId,
|
|
578
|
+
knowledgeDocumentVersionId: currentVersion.id,
|
|
579
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
580
|
+
})
|
|
581
|
+
: null;
|
|
582
|
+
if (
|
|
583
|
+
currentVersion &&
|
|
584
|
+
currentVersion.documentId &&
|
|
585
|
+
observedProviderRevision === entry.externalVersionId &&
|
|
586
|
+
observedMetadataHash === metadataHash &&
|
|
587
|
+
currentVersion.aclGeneration === aclGeneration &&
|
|
588
|
+
observedSourceLifecycleGeneration === resolved.source.lifecycleGeneration &&
|
|
589
|
+
observedObjectLifecycleGeneration === object.lifecycleGeneration &&
|
|
590
|
+
currentObligation?.status !== "invalidated"
|
|
591
|
+
) {
|
|
592
|
+
await updateKnowledgeSourceDocumentObservationMetadata(db, {
|
|
593
|
+
accountId: input.accountId,
|
|
594
|
+
workspaceId: input.workspaceId,
|
|
595
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
596
|
+
sourceId: input.sourceId,
|
|
597
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
598
|
+
objectId: object.id,
|
|
599
|
+
expectedObjectLifecycleGeneration: object.lifecycleGeneration,
|
|
600
|
+
versionId: currentVersion.id,
|
|
601
|
+
documentId: currentVersion.documentId,
|
|
602
|
+
title: entry.title,
|
|
603
|
+
sourceUri: entry.sourceUri,
|
|
604
|
+
sourceVersion: entry.externalVersionId ?? currentVersion.contentSha256,
|
|
605
|
+
sourceUpdatedAt: entry.modifiedTime,
|
|
606
|
+
syncObservation,
|
|
607
|
+
});
|
|
608
|
+
const obligation = await ensureVersionIndexed({
|
|
609
|
+
entry,
|
|
610
|
+
object,
|
|
611
|
+
version: currentVersion,
|
|
612
|
+
indexRequired: true,
|
|
613
|
+
obligation: currentObligation,
|
|
614
|
+
});
|
|
615
|
+
summary.unchanged += 1;
|
|
616
|
+
outcomes.push({
|
|
617
|
+
externalObjectId: entry.externalObjectId,
|
|
618
|
+
outcome: "unchanged" as const,
|
|
619
|
+
contentSha256: currentVersion.contentSha256,
|
|
620
|
+
providerRevision: entry.externalVersionId,
|
|
621
|
+
metadataHash,
|
|
622
|
+
aclEligibility: "pending" as const,
|
|
623
|
+
indexObligationId: obligation.id,
|
|
624
|
+
});
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
summary.phase = "transfer";
|
|
628
|
+
const bytes = await driver.fetchContent(entry, action.limits.maxFileBytes);
|
|
629
|
+
const contentSha256 = createHash("sha256").update(bytes).digest("hex");
|
|
630
|
+
const observationKey = createHash("sha256")
|
|
631
|
+
.update(
|
|
632
|
+
JSON.stringify({
|
|
633
|
+
externalObjectId: entry.externalObjectId,
|
|
634
|
+
providerRevision: entry.externalVersionId,
|
|
635
|
+
metadataHash,
|
|
636
|
+
contentSha256,
|
|
637
|
+
aclGeneration,
|
|
638
|
+
sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
639
|
+
objectLifecycleGeneration: object.lifecycleGeneration,
|
|
640
|
+
}),
|
|
641
|
+
)
|
|
642
|
+
.digest("hex");
|
|
643
|
+
if (
|
|
644
|
+
currentVersion?.contentSha256 === contentSha256 &&
|
|
645
|
+
currentVersion.documentId &&
|
|
646
|
+
currentVersion.fileId
|
|
647
|
+
) {
|
|
648
|
+
const version = await appendKnowledgeDocumentVersion(db, {
|
|
649
|
+
accountId: input.accountId,
|
|
650
|
+
workspaceId: input.workspaceId,
|
|
651
|
+
objectId: object.id,
|
|
652
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
653
|
+
expectedObjectLifecycleGeneration: object.lifecycleGeneration,
|
|
654
|
+
expectedVersionGeneration: object.versionGeneration,
|
|
655
|
+
externalVersionId: entry.externalVersionId ?? contentSha256,
|
|
656
|
+
contentSha256,
|
|
657
|
+
ingestionKey: `${entry.externalObjectId}:${observationKey}`,
|
|
658
|
+
sourceCursor: null,
|
|
659
|
+
sourceMetadata: {
|
|
660
|
+
title: entry.title,
|
|
661
|
+
mimeType: entry.mimeType,
|
|
662
|
+
driveId: entry.driveId,
|
|
663
|
+
providerRevision: entry.externalVersionId,
|
|
664
|
+
metadataHash,
|
|
665
|
+
aclEligibility: "pending",
|
|
666
|
+
sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
667
|
+
objectLifecycleGeneration: object.lifecycleGeneration,
|
|
668
|
+
},
|
|
669
|
+
sourceCreatedAt: entry.createdTime,
|
|
670
|
+
sourceUpdatedAt: entry.modifiedTime,
|
|
671
|
+
aclVersionId: acl.id,
|
|
672
|
+
aclGeneration: aclGeneration!,
|
|
673
|
+
documentId: currentVersion.documentId,
|
|
674
|
+
fileId: currentVersion.fileId,
|
|
675
|
+
locationMetadata: { sourceUri: entry.sourceUri },
|
|
676
|
+
documentObservationMetadata: {
|
|
677
|
+
title: entry.title,
|
|
678
|
+
sourceUri: entry.sourceUri,
|
|
679
|
+
sourceVersion: entry.externalVersionId ?? contentSha256,
|
|
680
|
+
sourceUpdatedAt: entry.modifiedTime,
|
|
681
|
+
},
|
|
682
|
+
syncObservation,
|
|
683
|
+
operationId: `document-observation:${object.id}:${observationKey}`,
|
|
684
|
+
reasonCode: "source_metadata_observed",
|
|
685
|
+
actor,
|
|
686
|
+
});
|
|
687
|
+
await updateKnowledgeSourceDocumentObservationMetadata(db, {
|
|
688
|
+
accountId: input.accountId,
|
|
689
|
+
workspaceId: input.workspaceId,
|
|
690
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
691
|
+
sourceId: input.sourceId,
|
|
692
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
693
|
+
objectId: object.id,
|
|
694
|
+
expectedObjectLifecycleGeneration: object.lifecycleGeneration,
|
|
695
|
+
versionId: version.id,
|
|
696
|
+
documentId: currentVersion.documentId,
|
|
697
|
+
title: entry.title,
|
|
698
|
+
sourceUri: entry.sourceUri,
|
|
699
|
+
sourceVersion: entry.externalVersionId ?? contentSha256,
|
|
700
|
+
sourceUpdatedAt: entry.modifiedTime,
|
|
701
|
+
syncObservation,
|
|
702
|
+
});
|
|
703
|
+
const obligation = await ensureVersionIndexed({
|
|
704
|
+
entry,
|
|
705
|
+
object,
|
|
706
|
+
version,
|
|
707
|
+
indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
|
|
708
|
+
});
|
|
709
|
+
summary.unchanged += 1;
|
|
710
|
+
summary.indexed += 1;
|
|
711
|
+
summary.aclPending += 1;
|
|
712
|
+
outcomes.push({
|
|
713
|
+
externalObjectId: entry.externalObjectId,
|
|
714
|
+
outcome: "unchanged" as const,
|
|
715
|
+
contentSha256,
|
|
716
|
+
sizeBytes: bytes.byteLength,
|
|
717
|
+
providerRevision: entry.externalVersionId,
|
|
718
|
+
metadataHash,
|
|
719
|
+
aclEligibility: "pending" as const,
|
|
720
|
+
indexObligationId: obligation.id,
|
|
721
|
+
});
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
const fileId = deterministicUuid(
|
|
725
|
+
`knowledge-blob:${input.accountId}:${input.workspaceId}:${contentSha256}`,
|
|
726
|
+
);
|
|
727
|
+
const objectKey = `workspaces/${input.workspaceId}/knowledge/blobs/${contentSha256}`;
|
|
728
|
+
await objectStorage.putObject({
|
|
729
|
+
key: objectKey,
|
|
730
|
+
contentType: entry.transfer.contentType,
|
|
731
|
+
body: bytes,
|
|
732
|
+
sha256: contentSha256,
|
|
733
|
+
});
|
|
734
|
+
const file = await ensureKnowledgeSourceBlobFile(db, {
|
|
735
|
+
accountId: input.accountId,
|
|
736
|
+
workspaceId: input.workspaceId,
|
|
737
|
+
fileId,
|
|
738
|
+
filename: entry.transfer.filename,
|
|
739
|
+
safeFilename: safeFilename(entry.transfer.filename),
|
|
740
|
+
contentType: entry.transfer.contentType,
|
|
741
|
+
sizeBytes: bytes.byteLength,
|
|
742
|
+
sha256: contentSha256,
|
|
743
|
+
bucket: objectStorage.bucket,
|
|
744
|
+
objectKey,
|
|
745
|
+
});
|
|
746
|
+
const document = await addDocumentToBase(db, {
|
|
747
|
+
accountId: input.accountId,
|
|
748
|
+
workspaceId: input.workspaceId,
|
|
749
|
+
baseId: base.id,
|
|
750
|
+
fileId: file.id,
|
|
751
|
+
title: entry.title,
|
|
752
|
+
sourceKind: "document",
|
|
753
|
+
sourceUri: entry.sourceUri,
|
|
754
|
+
sourceExternalId: entry.externalObjectId,
|
|
755
|
+
sourceTitle: entry.title,
|
|
756
|
+
sourceCreatedAt: entry.createdTime ?? undefined,
|
|
757
|
+
sourceUpdatedAt: entry.modifiedTime ?? undefined,
|
|
758
|
+
sourceVersion: entry.externalVersionId ?? contentSha256,
|
|
759
|
+
knowledgeSourceIdentity: deterministicUuid(
|
|
760
|
+
`knowledge-source-version:${object.id}:${observationKey}`,
|
|
761
|
+
),
|
|
762
|
+
authorityKind: action.destination.kind,
|
|
763
|
+
createdBy: action.initiatingSubjectId,
|
|
764
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
765
|
+
organizationAuthorityGranted: action.destination.kind === "organization",
|
|
766
|
+
agentAccess: false,
|
|
767
|
+
curationStatus: "none",
|
|
768
|
+
access: { viewerSubjectId: action.initiatingSubjectId },
|
|
769
|
+
});
|
|
770
|
+
const version = await appendKnowledgeDocumentVersion(db, {
|
|
771
|
+
accountId: input.accountId,
|
|
772
|
+
workspaceId: input.workspaceId,
|
|
773
|
+
objectId: object.id,
|
|
774
|
+
expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
775
|
+
expectedObjectLifecycleGeneration: object.lifecycleGeneration,
|
|
776
|
+
expectedVersionGeneration: object.versionGeneration,
|
|
777
|
+
externalVersionId: entry.externalVersionId ?? contentSha256,
|
|
778
|
+
contentSha256,
|
|
779
|
+
ingestionKey: `${entry.externalObjectId}:${observationKey}`,
|
|
780
|
+
sourceCursor: null,
|
|
781
|
+
sourceMetadata: {
|
|
782
|
+
title: entry.title,
|
|
783
|
+
mimeType: entry.mimeType,
|
|
784
|
+
driveId: entry.driveId,
|
|
785
|
+
providerRevision: entry.externalVersionId,
|
|
786
|
+
metadataHash,
|
|
787
|
+
aclEligibility: "pending",
|
|
788
|
+
sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
|
|
789
|
+
objectLifecycleGeneration: object.lifecycleGeneration,
|
|
790
|
+
},
|
|
791
|
+
sourceCreatedAt: entry.createdTime,
|
|
792
|
+
sourceUpdatedAt: entry.modifiedTime,
|
|
793
|
+
aclVersionId: acl.id,
|
|
794
|
+
aclGeneration: aclGeneration!,
|
|
795
|
+
documentId: document.id,
|
|
796
|
+
fileId: file.id,
|
|
797
|
+
locationMetadata: { sourceUri: entry.sourceUri },
|
|
798
|
+
syncObservation,
|
|
799
|
+
operationId: `document-version:${object.id}:${observationKey}`,
|
|
800
|
+
reasonCode: "source_content_observed",
|
|
801
|
+
actor,
|
|
802
|
+
});
|
|
803
|
+
const obligation = await ensureVersionIndexed({
|
|
804
|
+
entry,
|
|
805
|
+
object,
|
|
806
|
+
version,
|
|
807
|
+
indexRequired: true,
|
|
808
|
+
});
|
|
809
|
+
summary.indexed += 1;
|
|
810
|
+
summary.imported += 1;
|
|
811
|
+
summary.aclPending += 1;
|
|
812
|
+
summary.bytes += bytes.byteLength;
|
|
813
|
+
outcomes.push({
|
|
814
|
+
externalObjectId: entry.externalObjectId,
|
|
815
|
+
outcome: "imported" as const,
|
|
816
|
+
contentSha256,
|
|
817
|
+
sizeBytes: bytes.byteLength,
|
|
818
|
+
providerRevision: entry.externalVersionId,
|
|
819
|
+
metadataHash,
|
|
820
|
+
aclEligibility: "pending" as const,
|
|
821
|
+
indexObligationId: obligation.id,
|
|
822
|
+
});
|
|
823
|
+
} catch (error) {
|
|
824
|
+
if (error instanceof KnowledgeSourceSyncObservationFenceError) {
|
|
825
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
826
|
+
}
|
|
827
|
+
const message = boundedError(error);
|
|
828
|
+
const itemFailure =
|
|
829
|
+
error instanceof SyncFailure
|
|
830
|
+
? error
|
|
831
|
+
: new SyncFailure("item_processing_failed", false);
|
|
832
|
+
summary.failed += 1;
|
|
833
|
+
summary.retryable ||= itemFailure.retryable;
|
|
834
|
+
if (summary.failures.length < action.limits.maxFailureDetails) {
|
|
835
|
+
summary.failures.push({
|
|
836
|
+
externalObjectId: entry.externalObjectId,
|
|
837
|
+
code: itemFailure.code,
|
|
838
|
+
retryable: itemFailure.retryable,
|
|
839
|
+
message,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
outcomes.push({
|
|
843
|
+
externalObjectId: entry.externalObjectId,
|
|
844
|
+
outcome: "failed" as const,
|
|
845
|
+
reasonCode: itemFailure.code,
|
|
846
|
+
detail: message,
|
|
847
|
+
providerRevision: entry.externalVersionId,
|
|
848
|
+
metadataHash: entryMetadataHash(entry),
|
|
849
|
+
aclEligibility: "pending" as const,
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
await recordKnowledgeSourceSyncItemOutcomes(db, {
|
|
854
|
+
accountId: input.accountId,
|
|
855
|
+
workspaceId: input.workspaceId,
|
|
856
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
857
|
+
knowledgeSyncRunId: knowledgeRun.id,
|
|
858
|
+
sourceId: input.sourceId,
|
|
859
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
860
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
861
|
+
outcomes,
|
|
862
|
+
});
|
|
863
|
+
|
|
864
|
+
if (inventory.status === "paused" && inventory.checkpoint) {
|
|
865
|
+
if (
|
|
866
|
+
inventory.stopReason === "provider_error" ||
|
|
867
|
+
inventory.stopReason === "incomplete_search"
|
|
868
|
+
) {
|
|
869
|
+
throw new SyncFailure(
|
|
870
|
+
inventory.stopReason === "provider_error"
|
|
871
|
+
? "provider_unavailable"
|
|
872
|
+
: "provider_payload_invalid",
|
|
873
|
+
inventory.stopReason === "provider_error",
|
|
874
|
+
);
|
|
875
|
+
}
|
|
876
|
+
if (inventory.hardLimitReached) {
|
|
877
|
+
summary.limitReached = inventoryLimit(inventory.stopReason);
|
|
878
|
+
throw new SyncFailure("resource_limit", false);
|
|
879
|
+
}
|
|
880
|
+
if (inventory.stopReason !== "elapsed_time_limit") {
|
|
881
|
+
summary.limitReached = inventoryLimit(inventory.stopReason);
|
|
882
|
+
throw new SyncFailure("resource_limit", false);
|
|
883
|
+
}
|
|
884
|
+
summary.checkpointed = true;
|
|
885
|
+
summary.phase = "checkpoint";
|
|
886
|
+
await checkpointKnowledgeSourceSync(db, {
|
|
887
|
+
accountId: input.accountId,
|
|
888
|
+
workspaceId: input.workspaceId,
|
|
889
|
+
sourceId: input.sourceId,
|
|
890
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
891
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
892
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
893
|
+
executionCheckpoint: executionCheckpoint as Record<string, unknown>,
|
|
894
|
+
observationFence: {
|
|
895
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
896
|
+
scanGeneration: liveState.activeScanGeneration,
|
|
897
|
+
executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
|
|
898
|
+
observations: observationFloors,
|
|
899
|
+
},
|
|
900
|
+
});
|
|
901
|
+
return { action: "continue" };
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
const tombstonedExternalObjectIds = inventory.authoritativeFullScan
|
|
905
|
+
? await reconcileKnowledgeSourceSyncCompleteScan(db, {
|
|
906
|
+
accountId: input.accountId,
|
|
907
|
+
workspaceId: input.workspaceId,
|
|
908
|
+
sourceId: input.sourceId,
|
|
909
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
910
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
911
|
+
sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
|
|
912
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
913
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
914
|
+
scanGeneration: liveState.activeScanGeneration,
|
|
915
|
+
})
|
|
916
|
+
: [];
|
|
917
|
+
if (tombstonedExternalObjectIds.length > 0) {
|
|
918
|
+
summary.skipped += tombstonedExternalObjectIds.length;
|
|
919
|
+
await recordKnowledgeSourceSyncItemOutcomes(db, {
|
|
920
|
+
accountId: input.accountId,
|
|
921
|
+
workspaceId: input.workspaceId,
|
|
922
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
923
|
+
knowledgeSyncRunId: knowledgeRun.id,
|
|
924
|
+
sourceId: input.sourceId,
|
|
925
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
926
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
927
|
+
outcomes: tombstonedExternalObjectIds.map((externalObjectId) => ({
|
|
928
|
+
externalObjectId,
|
|
929
|
+
outcome: "tombstoned" as const,
|
|
930
|
+
reasonCode: "authoritative_scan_absent",
|
|
931
|
+
aclEligibility: "denied" as const,
|
|
932
|
+
})),
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
await completeKnowledgeSyncRun(db, {
|
|
937
|
+
accountId: input.accountId,
|
|
938
|
+
workspaceId: input.workspaceId,
|
|
939
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
940
|
+
runId: knowledgeRun.id,
|
|
941
|
+
state: "succeeded",
|
|
942
|
+
outputCursor: null,
|
|
943
|
+
watermark: new Date().toISOString(),
|
|
944
|
+
metadata: { ...summary, providerCursor: inventory.providerCursor },
|
|
945
|
+
reasonCode: "scheduled_source_sync",
|
|
946
|
+
});
|
|
947
|
+
summary.phase = "completed";
|
|
948
|
+
summary.elapsedMs = Date.now() - startedAt;
|
|
949
|
+
const settled = await settleKnowledgeSourceSyncLease(db, {
|
|
950
|
+
accountId: input.accountId,
|
|
951
|
+
workspaceId: input.workspaceId,
|
|
952
|
+
sourceId: input.sourceId,
|
|
953
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
954
|
+
knowledgeSyncRunId: knowledgeRun.id,
|
|
955
|
+
status: "succeeded",
|
|
956
|
+
summary,
|
|
957
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
958
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
959
|
+
sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
|
|
960
|
+
completedSourceSyncGeneration: knowledgeRun.inputSyncGeneration + 1,
|
|
961
|
+
providerCursor: inventory.providerCursor,
|
|
962
|
+
observationFence: {
|
|
963
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
964
|
+
scanGeneration: liveState.activeScanGeneration,
|
|
965
|
+
executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
|
|
966
|
+
observations: observationFloors,
|
|
967
|
+
},
|
|
968
|
+
});
|
|
969
|
+
await recordConnectorUsage("completed", summary.scanned, "item");
|
|
970
|
+
await recordConnectorUsage("items", summary.imported, "item");
|
|
971
|
+
await recordConnectorUsage("bytes", summary.bytes, "byte");
|
|
972
|
+
observability.incrementCounter({
|
|
973
|
+
name: "opengeni_knowledge_source_sync_runs_total",
|
|
974
|
+
labels: { provider: providerLabel, outcome: "succeeded" },
|
|
975
|
+
});
|
|
976
|
+
observability.incrementCounter({
|
|
977
|
+
name: "opengeni_knowledge_source_sync_items_total",
|
|
978
|
+
labels: { provider: providerLabel, outcome: "imported" },
|
|
979
|
+
amount: summary.imported,
|
|
980
|
+
});
|
|
981
|
+
for (const outcome of ["unchanged", "skipped", "failed"] as const) {
|
|
982
|
+
observability.incrementCounter({
|
|
983
|
+
name: "opengeni_knowledge_source_sync_items_total",
|
|
984
|
+
labels: { provider: providerLabel, outcome },
|
|
985
|
+
amount: summary[outcome],
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
observability.incrementCounter({
|
|
989
|
+
name: "opengeni_knowledge_source_sync_bytes_total",
|
|
990
|
+
labels: { provider: providerLabel },
|
|
991
|
+
amount: summary.bytes,
|
|
992
|
+
});
|
|
993
|
+
return {
|
|
994
|
+
action: "complete",
|
|
995
|
+
bufferedWake: settled.bufferedWake,
|
|
996
|
+
bufferedScheduledTaskRunId: settled.bufferedScheduledTaskRunId,
|
|
997
|
+
};
|
|
998
|
+
} catch (error) {
|
|
999
|
+
const failure =
|
|
1000
|
+
error instanceof SyncFailure ? error : new SyncFailure("internal_failure", false);
|
|
1001
|
+
summary.phase = "failed";
|
|
1002
|
+
summary.retryable ||= failure.retryable;
|
|
1003
|
+
summary.elapsedMs = Date.now() - startedAt;
|
|
1004
|
+
summary.reconnectRequired = failure.reconnectRequired;
|
|
1005
|
+
summary.failed = Math.max(summary.failed, 1);
|
|
1006
|
+
if (summary.failures.length === 0) {
|
|
1007
|
+
summary.failures.push({
|
|
1008
|
+
externalObjectId: input.sourceId,
|
|
1009
|
+
code: failure.code,
|
|
1010
|
+
retryable: failure.retryable,
|
|
1011
|
+
message: boundedError(error),
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
if (knowledgeRunId) {
|
|
1015
|
+
await completeKnowledgeSyncRun(db, {
|
|
1016
|
+
accountId: input.accountId,
|
|
1017
|
+
workspaceId: input.workspaceId,
|
|
1018
|
+
initiatingSubjectId: task.action.initiatingSubjectId,
|
|
1019
|
+
runId: knowledgeRunId,
|
|
1020
|
+
state: "failed",
|
|
1021
|
+
metadata: summary,
|
|
1022
|
+
errorCode: failure.code,
|
|
1023
|
+
reasonCode: "scheduled_source_sync_failed",
|
|
1024
|
+
}).catch(() => undefined);
|
|
1025
|
+
}
|
|
1026
|
+
const settlementInput: Parameters<typeof settleKnowledgeSourceSyncLease>[1] = {
|
|
1027
|
+
accountId: input.accountId,
|
|
1028
|
+
workspaceId: input.workspaceId,
|
|
1029
|
+
sourceId: input.sourceId,
|
|
1030
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
1031
|
+
knowledgeSyncRunId: knowledgeRunId,
|
|
1032
|
+
status: "failed",
|
|
1033
|
+
summary,
|
|
1034
|
+
error: failure.code,
|
|
1035
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
1036
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
1037
|
+
sourceSyncGeneration: activeSourceSyncGeneration ?? lease.state.sourceSyncGeneration,
|
|
1038
|
+
// Failed or incomplete provider walks are not authoritative scans.
|
|
1039
|
+
// A later scheduled run must restart from the root under a fresh
|
|
1040
|
+
// observation generation before it can infer object absence.
|
|
1041
|
+
executionCheckpoint: null,
|
|
1042
|
+
};
|
|
1043
|
+
const settled = failure.reconnectRequired
|
|
1044
|
+
? await deauthorizeAndSettleKnowledgeSourceSyncFailure(db, {
|
|
1045
|
+
deauthorization: {
|
|
1046
|
+
accountId: input.accountId,
|
|
1047
|
+
workspaceId: input.workspaceId,
|
|
1048
|
+
sourceId: input.sourceId,
|
|
1049
|
+
audience: action.destination,
|
|
1050
|
+
operationId: `knowledge-sync-deauthorize:${input.sourceId}:${action.connection.connectionVersion}:${failure.code}`,
|
|
1051
|
+
reasonCode: "connection_reconnect_required",
|
|
1052
|
+
actor: {
|
|
1053
|
+
kind: "service",
|
|
1054
|
+
subjectId: "knowledge-source-sync",
|
|
1055
|
+
initiatingHumanSubjectId: action.initiatingSubjectId,
|
|
1056
|
+
},
|
|
1057
|
+
},
|
|
1058
|
+
settlement: settlementInput,
|
|
1059
|
+
})
|
|
1060
|
+
: await settleKnowledgeSourceSyncLease(db, settlementInput);
|
|
1061
|
+
await recordConnectorUsage("completed", summary.scanned, "item");
|
|
1062
|
+
await recordConnectorUsage("items", summary.imported, "item");
|
|
1063
|
+
await recordConnectorUsage("bytes", summary.bytes, "byte");
|
|
1064
|
+
observability.incrementCounter({
|
|
1065
|
+
name: "opengeni_knowledge_source_sync_runs_total",
|
|
1066
|
+
labels: { provider: providerLabel, outcome: "failed" },
|
|
1067
|
+
});
|
|
1068
|
+
for (const outcome of ["imported", "unchanged", "skipped", "failed"] as const) {
|
|
1069
|
+
observability.incrementCounter({
|
|
1070
|
+
name: "opengeni_knowledge_source_sync_items_total",
|
|
1071
|
+
labels: { provider: providerLabel, outcome },
|
|
1072
|
+
amount: summary[outcome],
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
observability.incrementCounter({
|
|
1076
|
+
name: "opengeni_knowledge_source_sync_bytes_total",
|
|
1077
|
+
labels: { provider: providerLabel },
|
|
1078
|
+
amount: summary.bytes,
|
|
1079
|
+
});
|
|
1080
|
+
return {
|
|
1081
|
+
action: "failed",
|
|
1082
|
+
bufferedWake: settled.bufferedWake,
|
|
1083
|
+
bufferedScheduledTaskRunId: settled.bufferedScheduledTaskRunId,
|
|
1084
|
+
};
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
async function failWithoutProvider(code: string): Promise<RunKnowledgeSourceSyncBatchResult> {
|
|
1088
|
+
const terminalSummary = {
|
|
1089
|
+
phase: "failed" as const,
|
|
1090
|
+
scanned: 0,
|
|
1091
|
+
imported: 0,
|
|
1092
|
+
unchanged: 0,
|
|
1093
|
+
skipped: 0,
|
|
1094
|
+
failed: 1,
|
|
1095
|
+
bytes: 0,
|
|
1096
|
+
providerRequests: 0,
|
|
1097
|
+
elapsedMs: 0,
|
|
1098
|
+
indexed: 0,
|
|
1099
|
+
aclPending: 0,
|
|
1100
|
+
retryable: false,
|
|
1101
|
+
limitReached: null,
|
|
1102
|
+
checkpointed: false,
|
|
1103
|
+
reconnectRequired: false,
|
|
1104
|
+
failures: [
|
|
1105
|
+
{
|
|
1106
|
+
externalObjectId: input.sourceId,
|
|
1107
|
+
code: "authority_changed" as const,
|
|
1108
|
+
retryable: false,
|
|
1109
|
+
message: code,
|
|
1110
|
+
},
|
|
1111
|
+
],
|
|
1112
|
+
};
|
|
1113
|
+
await updateScheduledTaskRun(db, input.workspaceId, input.scheduledTaskRunId, {
|
|
1114
|
+
status: "failed",
|
|
1115
|
+
actionKind: "knowledge_source_sync",
|
|
1116
|
+
knowledgeSummary: terminalSummary,
|
|
1117
|
+
completedAt: new Date(),
|
|
1118
|
+
error: code,
|
|
1119
|
+
}).catch(() => undefined);
|
|
1120
|
+
await completeKnowledgeSourceSyncWakeWithoutLease(db, {
|
|
1121
|
+
accountId: input.accountId,
|
|
1122
|
+
workspaceId: input.workspaceId,
|
|
1123
|
+
sourceId: input.sourceId,
|
|
1124
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
1125
|
+
}).catch(() => undefined);
|
|
1126
|
+
await recordConnectorUsage("completed", 0, "item").catch(() => undefined);
|
|
1127
|
+
observability.incrementCounter({
|
|
1128
|
+
name: "opengeni_knowledge_source_sync_runs_total",
|
|
1129
|
+
labels: { provider: "unknown", outcome: "failed_preflight" },
|
|
1130
|
+
});
|
|
1131
|
+
return { action: "failed", bufferedWake: false };
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
async function recordConnectorUsage(
|
|
1135
|
+
kind: "completed" | "items" | "bytes",
|
|
1136
|
+
quantity: number,
|
|
1137
|
+
unit: string,
|
|
1138
|
+
) {
|
|
1139
|
+
if (quantity === 0 && kind !== "completed") return;
|
|
1140
|
+
await recordUsageEvent(db, {
|
|
1141
|
+
accountId: input.accountId,
|
|
1142
|
+
workspaceId: input.workspaceId,
|
|
1143
|
+
eventType: `knowledge_source_sync.${kind}`,
|
|
1144
|
+
quantity: kind === "completed" ? 1 : quantity,
|
|
1145
|
+
unit,
|
|
1146
|
+
sourceResourceType: "scheduled_task_run",
|
|
1147
|
+
sourceResourceId: input.scheduledTaskRunId,
|
|
1148
|
+
initiator: { kind: "service", subjectId: "knowledge-source-sync" },
|
|
1149
|
+
initiatorContext: {
|
|
1150
|
+
scheduledTaskId: input.taskId,
|
|
1151
|
+
scheduledTaskRunId: input.scheduledTaskRunId,
|
|
1152
|
+
},
|
|
1153
|
+
origin: "scheduled_task",
|
|
1154
|
+
idempotencyKey: `usage:knowledge_source_sync.${kind}:${input.scheduledTaskRunId}`,
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
},
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* Permission loss must fail closed, while the exact source lease must never be
|
|
1163
|
+
* stranded by a deauthorization error. If deauthorization fails, release the
|
|
1164
|
+
* lease without settling the wake/run and surface the error so Temporal retries
|
|
1165
|
+
* this DB-only safety obligation. Terminal settlement is allowed only after
|
|
1166
|
+
* deauthorization succeeds.
|
|
1167
|
+
*/
|
|
1168
|
+
export async function deauthorizeAndSettleKnowledgeSourceSyncFailure(
|
|
1169
|
+
db: Database,
|
|
1170
|
+
input: {
|
|
1171
|
+
deauthorization: Parameters<typeof deauthorizeKnowledgeSourceRetrieval>[1];
|
|
1172
|
+
settlement: Parameters<typeof settleKnowledgeSourceSyncLease>[1];
|
|
1173
|
+
},
|
|
1174
|
+
): Promise<Awaited<ReturnType<typeof settleKnowledgeSourceSyncLease>>> {
|
|
1175
|
+
try {
|
|
1176
|
+
await deauthorizeKnowledgeSourceRetrieval(db, input.deauthorization);
|
|
1177
|
+
} catch (error) {
|
|
1178
|
+
await releaseKnowledgeSourceSyncLeaseForRetry(db, {
|
|
1179
|
+
accountId: input.settlement.accountId,
|
|
1180
|
+
workspaceId: input.settlement.workspaceId,
|
|
1181
|
+
sourceId: input.settlement.sourceId,
|
|
1182
|
+
scheduledTaskRunId: input.settlement.scheduledTaskRunId,
|
|
1183
|
+
});
|
|
1184
|
+
throw error;
|
|
1185
|
+
}
|
|
1186
|
+
return await settleKnowledgeSourceSyncLease(db, input.settlement);
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
export type GoogleDriveSyncProviderPort = {
|
|
1190
|
+
now?: () => number;
|
|
1191
|
+
getStartPageToken: (driveId: string | null) => Promise<string>;
|
|
1192
|
+
listChanges: (
|
|
1193
|
+
pageToken: string,
|
|
1194
|
+
driveId: string | null,
|
|
1195
|
+
pageSize: number,
|
|
1196
|
+
) => Promise<GoogleDriveChangesPage>;
|
|
1197
|
+
getFile: (fileId: string) => Promise<GoogleDriveInventoryProviderItem | null>;
|
|
1198
|
+
listChildren: (input: {
|
|
1199
|
+
folderId: string;
|
|
1200
|
+
driveId: string | null;
|
|
1201
|
+
pageToken: string | null;
|
|
1202
|
+
pageSize: number;
|
|
1203
|
+
}) => Promise<GoogleDriveInventoryPage>;
|
|
1204
|
+
};
|
|
1205
|
+
|
|
1206
|
+
async function resolveKnowledgeSyncProvider(input: {
|
|
1207
|
+
db: Database;
|
|
1208
|
+
settings: Settings;
|
|
1209
|
+
accountId: string;
|
|
1210
|
+
workspaceId: string;
|
|
1211
|
+
action: Extract<ScheduledTask["action"], { kind: "knowledge_source_sync" }>;
|
|
1212
|
+
sourceId: string;
|
|
1213
|
+
externalSourceId: string;
|
|
1214
|
+
}): Promise<{
|
|
1215
|
+
kind: "google_drive" | "atlassian";
|
|
1216
|
+
selectedDestination: NonNullable<
|
|
1217
|
+
GoogleDriveSelectedSource["destination"] | AtlassianSelectedSource["destination"]
|
|
1218
|
+
>;
|
|
1219
|
+
driver: KnowledgeSourceSyncDriver<KnowledgeSyncEntry, KnowledgeSyncStopReason>;
|
|
1220
|
+
}> {
|
|
1221
|
+
const { action } = input;
|
|
1222
|
+
const connection = await getConnectionMetadata(
|
|
1223
|
+
input.db,
|
|
1224
|
+
input.workspaceId,
|
|
1225
|
+
action.connection.connectionId,
|
|
1226
|
+
action.initiatingSubjectId,
|
|
1227
|
+
);
|
|
1228
|
+
if (
|
|
1229
|
+
!connection ||
|
|
1230
|
+
connection.accountId !== input.accountId ||
|
|
1231
|
+
connection.subjectId !== action.connection.ownerSubjectId ||
|
|
1232
|
+
connection.providerDomain.toLowerCase() !== action.connection.providerDomain.toLowerCase() ||
|
|
1233
|
+
connection.kind !== action.connection.kind ||
|
|
1234
|
+
connection.version < action.connection.connectionVersion ||
|
|
1235
|
+
connection.status !== "active"
|
|
1236
|
+
) {
|
|
1237
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
const googleInitial =
|
|
1241
|
+
connection.providerDomain === GOOGLE_DRIVE_PROVIDER_DOMAIN
|
|
1242
|
+
? GoogleDriveConnectionMetadata.safeParse(connection.metadata)
|
|
1243
|
+
: null;
|
|
1244
|
+
const atlassianInitial =
|
|
1245
|
+
connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN
|
|
1246
|
+
? AtlassianConnectionMetadata.safeParse(connection.metadata)
|
|
1247
|
+
: null;
|
|
1248
|
+
const initialAtlassianMetadata = atlassianInitial?.success ? atlassianInitial.data : null;
|
|
1249
|
+
if (!googleInitial?.success && !initialAtlassianMetadata) {
|
|
1250
|
+
throw new SyncFailure("provider_rejected", false);
|
|
1251
|
+
}
|
|
1252
|
+
if (
|
|
1253
|
+
googleInitial?.success &&
|
|
1254
|
+
!googleDriveScopesAllowCapability(connection.grantedScopes, "recursive_source_sync")
|
|
1255
|
+
) {
|
|
1256
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1257
|
+
}
|
|
1258
|
+
if (atlassianInitial?.success && !atlassianScopesAllowRead(connection.grantedScopes)) {
|
|
1259
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1260
|
+
}
|
|
1261
|
+
const initialLifecycle = googleInitial?.success
|
|
1262
|
+
? googleInitial.data.lifecycle
|
|
1263
|
+
: initialAtlassianMetadata!.lifecycle;
|
|
1264
|
+
if (initialLifecycle?.state && initialLifecycle.state !== "active") {
|
|
1265
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1266
|
+
}
|
|
1267
|
+
const initialSource = googleInitial?.success
|
|
1268
|
+
? selectedDriveSource(googleInitial.data, input.externalSourceId)
|
|
1269
|
+
: selectedAtlassianSource(initialAtlassianMetadata!, input.externalSourceId);
|
|
1270
|
+
if (
|
|
1271
|
+
!initialSource ||
|
|
1272
|
+
!initialSource.syncEnabled ||
|
|
1273
|
+
initialSource.configGeneration !== action.sourceConfigGeneration ||
|
|
1274
|
+
!initialSource.destination ||
|
|
1275
|
+
scopedKnowledgeScopeKey(
|
|
1276
|
+
googleInitial?.success
|
|
1277
|
+
? googleDriveKnowledgeScope(initialSource.destination)
|
|
1278
|
+
: atlassianKnowledgeScope(initialSource.destination),
|
|
1279
|
+
) !== scopedKnowledgeScopeKey(action.destination)
|
|
1280
|
+
) {
|
|
1281
|
+
throw new SyncFailure("authority_changed", false);
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
const token = await buildConnectionTokenResolver(
|
|
1285
|
+
input.db,
|
|
1286
|
+
input.settings,
|
|
1287
|
+
)({
|
|
1288
|
+
workspaceId: input.workspaceId,
|
|
1289
|
+
subjectId: action.initiatingSubjectId,
|
|
1290
|
+
serverId: "knowledge-source-sync",
|
|
1291
|
+
connectionRef: {
|
|
1292
|
+
connectionId: connection.id,
|
|
1293
|
+
providerDomain: connection.providerDomain,
|
|
1294
|
+
kind: connection.kind,
|
|
1295
|
+
subjectScope: "subject",
|
|
1296
|
+
},
|
|
1297
|
+
destinationUrl:
|
|
1298
|
+
connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN
|
|
1299
|
+
? ATLASSIAN_API_BASE
|
|
1300
|
+
: "https://www.googleapis.com",
|
|
1301
|
+
});
|
|
1302
|
+
if (token.status !== "ok") {
|
|
1303
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1304
|
+
}
|
|
1305
|
+
const resolvedConnection = await getConnectionMetadata(
|
|
1306
|
+
input.db,
|
|
1307
|
+
input.workspaceId,
|
|
1308
|
+
action.connection.connectionId,
|
|
1309
|
+
action.initiatingSubjectId,
|
|
1310
|
+
);
|
|
1311
|
+
if (
|
|
1312
|
+
!resolvedConnection ||
|
|
1313
|
+
resolvedConnection.accountId !== input.accountId ||
|
|
1314
|
+
resolvedConnection.subjectId !== action.connection.ownerSubjectId ||
|
|
1315
|
+
resolvedConnection.providerDomain.toLowerCase() !==
|
|
1316
|
+
action.connection.providerDomain.toLowerCase() ||
|
|
1317
|
+
resolvedConnection.kind !== action.connection.kind ||
|
|
1318
|
+
resolvedConnection.version !== token.connectionVersion ||
|
|
1319
|
+
resolvedConnection.status !== "active"
|
|
1320
|
+
) {
|
|
1321
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1322
|
+
}
|
|
1323
|
+
const authorization = token.headers.Authorization ?? token.headers.authorization;
|
|
1324
|
+
if (resolvedConnection.providerDomain === GOOGLE_DRIVE_PROVIDER_DOMAIN) {
|
|
1325
|
+
const metadata = GoogleDriveConnectionMetadata.parse(resolvedConnection.metadata);
|
|
1326
|
+
if (
|
|
1327
|
+
!googleDriveSyncProviderAccessAllowed({
|
|
1328
|
+
connectionVersion: resolvedConnection.version,
|
|
1329
|
+
resolvedConnectionVersion: token.connectionVersion,
|
|
1330
|
+
connectionStatus: resolvedConnection.status,
|
|
1331
|
+
lifecycleState: metadata.lifecycle?.state,
|
|
1332
|
+
grantedScopes: resolvedConnection.grantedScopes,
|
|
1333
|
+
})
|
|
1334
|
+
) {
|
|
1335
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1336
|
+
}
|
|
1337
|
+
const source = selectedDriveSource(metadata, input.externalSourceId);
|
|
1338
|
+
if (!validSelectedSource(source, action, googleDriveKnowledgeScope)) {
|
|
1339
|
+
throw new SyncFailure("authority_changed", false);
|
|
1340
|
+
}
|
|
1341
|
+
return {
|
|
1342
|
+
kind: "google_drive",
|
|
1343
|
+
selectedDestination: source.destination!,
|
|
1344
|
+
driver: googleDriveSyncDriver({
|
|
1345
|
+
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1346
|
+
metadata,
|
|
1347
|
+
selectedSource: source,
|
|
1348
|
+
accountId: input.accountId,
|
|
1349
|
+
workspaceId: input.workspaceId,
|
|
1350
|
+
initiatingSubjectId: action.initiatingSubjectId,
|
|
1351
|
+
authorization,
|
|
1352
|
+
limits: action.limits,
|
|
1353
|
+
connectionId: resolvedConnection.id,
|
|
1354
|
+
fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
|
|
1355
|
+
observedExternalObjectIds: async (externalObjectIds) =>
|
|
1356
|
+
await listObservedKnowledgeSourceSyncExternalObjectIds(input.db, {
|
|
1357
|
+
accountId: input.accountId,
|
|
1358
|
+
workspaceId: input.workspaceId,
|
|
1359
|
+
sourceId: input.sourceId,
|
|
1360
|
+
externalObjectIds,
|
|
1361
|
+
}),
|
|
1362
|
+
}) as unknown as KnowledgeSourceSyncDriver<KnowledgeSyncEntry, KnowledgeSyncStopReason>,
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1365
|
+
if (resolvedConnection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN) {
|
|
1366
|
+
const metadata = AtlassianConnectionMetadata.parse(resolvedConnection.metadata);
|
|
1367
|
+
if (
|
|
1368
|
+
metadata.lifecycle?.state !== "active" ||
|
|
1369
|
+
!atlassianScopesAllowRead(resolvedConnection.grantedScopes)
|
|
1370
|
+
) {
|
|
1371
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
1372
|
+
}
|
|
1373
|
+
const source = selectedAtlassianSource(metadata, input.externalSourceId);
|
|
1374
|
+
if (!validSelectedSource(source, action, atlassianKnowledgeScope)) {
|
|
1375
|
+
throw new SyncFailure("authority_changed", false);
|
|
1376
|
+
}
|
|
1377
|
+
return {
|
|
1378
|
+
kind: "atlassian",
|
|
1379
|
+
selectedDestination: source.destination!,
|
|
1380
|
+
driver: atlassianSyncDriver({
|
|
1381
|
+
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1382
|
+
metadata,
|
|
1383
|
+
selectedSource: source,
|
|
1384
|
+
authorization,
|
|
1385
|
+
limits: action.limits,
|
|
1386
|
+
}) as unknown as KnowledgeSourceSyncDriver<KnowledgeSyncEntry, KnowledgeSyncStopReason>,
|
|
1387
|
+
};
|
|
1388
|
+
}
|
|
1389
|
+
throw new SyncFailure("provider_rejected", false);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
function validSelectedSource<Source extends GoogleDriveSelectedSource | AtlassianSelectedSource>(
|
|
1393
|
+
source: Source | null,
|
|
1394
|
+
action: Extract<ScheduledTask["action"], { kind: "knowledge_source_sync" }>,
|
|
1395
|
+
scope: (destination: NonNullable<Source["destination"]>) => ScopedKnowledgeScope,
|
|
1396
|
+
): source is Source & { destination: NonNullable<Source["destination"]> } {
|
|
1397
|
+
return Boolean(
|
|
1398
|
+
source &&
|
|
1399
|
+
source.syncEnabled &&
|
|
1400
|
+
source.configGeneration === action.sourceConfigGeneration &&
|
|
1401
|
+
source.destination &&
|
|
1402
|
+
scopedKnowledgeScopeKey(scope(source.destination)) ===
|
|
1403
|
+
scopedKnowledgeScopeKey(action.destination),
|
|
1404
|
+
);
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
function selectedAtlassianSource(
|
|
1408
|
+
metadata: ReturnType<typeof AtlassianConnectionMetadata.parse>,
|
|
1409
|
+
externalSourceId: string,
|
|
1410
|
+
): AtlassianSelectedSource | null {
|
|
1411
|
+
return metadata.selectedSources.find((source) => source.id === externalSourceId) ?? null;
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
function atlassianSyncDriver(input: {
|
|
1415
|
+
actionProviderCoordinationKey: string;
|
|
1416
|
+
metadata: ReturnType<typeof AtlassianConnectionMetadata.parse>;
|
|
1417
|
+
selectedSource: AtlassianSelectedSource;
|
|
1418
|
+
authorization: string | undefined;
|
|
1419
|
+
limits: {
|
|
1420
|
+
maxItems: number;
|
|
1421
|
+
maxBytes: number;
|
|
1422
|
+
maxFileBytes: number;
|
|
1423
|
+
maxProviderRequests: number;
|
|
1424
|
+
maxElapsedSeconds: number;
|
|
1425
|
+
};
|
|
1426
|
+
}): KnowledgeSourceSyncDriver<AtlassianInventoryEntry, AtlassianInventoryStopReason> {
|
|
1427
|
+
return {
|
|
1428
|
+
providerKey: "atlassian",
|
|
1429
|
+
providerDomain: ATLASSIAN_PROVIDER_DOMAIN,
|
|
1430
|
+
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
1431
|
+
inventory: async (checkpoint) => {
|
|
1432
|
+
const inventory = await inventoryAtlassianSource({
|
|
1433
|
+
cloudId: input.selectedSource.cloudId,
|
|
1434
|
+
source: input.selectedSource,
|
|
1435
|
+
limits: {
|
|
1436
|
+
maxItems: input.limits.maxItems,
|
|
1437
|
+
maxApiRequests: input.limits.maxProviderRequests,
|
|
1438
|
+
maxElapsedMs: input.limits.maxElapsedSeconds * 1_000,
|
|
1439
|
+
pageSize: Math.min(input.limits.maxItems, 100),
|
|
1440
|
+
},
|
|
1441
|
+
checkpoint,
|
|
1442
|
+
listPage: async (cursor, pageSize) =>
|
|
1443
|
+
await listAtlassianItems(input.selectedSource, cursor, pageSize, input.authorization),
|
|
1444
|
+
});
|
|
1445
|
+
return {
|
|
1446
|
+
...inventory,
|
|
1447
|
+
providerCursor: null,
|
|
1448
|
+
authoritativeFullScan: inventory.status === "complete",
|
|
1449
|
+
cursorInvalidated: false,
|
|
1450
|
+
hardLimitReached:
|
|
1451
|
+
inventory.status === "paused" &&
|
|
1452
|
+
inventory.stopReason !== "elapsed_time_limit" &&
|
|
1453
|
+
inventory.stopReason !== "provider_error",
|
|
1454
|
+
};
|
|
1455
|
+
},
|
|
1456
|
+
fetchContent: async (entry, maxBytes) =>
|
|
1457
|
+
await fetchAtlassianContent(input.selectedSource, entry, input.authorization, maxBytes),
|
|
1458
|
+
citationLocator: (entry) => ({
|
|
1459
|
+
version: 1,
|
|
1460
|
+
providerKey: "atlassian",
|
|
1461
|
+
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
1462
|
+
cloudId: input.selectedSource.cloudId,
|
|
1463
|
+
sourceKind: input.selectedSource.kind,
|
|
1464
|
+
externalObjectId: entry.externalObjectId,
|
|
1465
|
+
sourceUri: entry.sourceUri,
|
|
1466
|
+
}),
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
export function googleDriveSyncDriver(input: {
|
|
1471
|
+
actionProviderCoordinationKey: string;
|
|
1472
|
+
metadata: ReturnType<typeof GoogleDriveConnectionMetadata.parse>;
|
|
1473
|
+
selectedSource: GoogleDriveSelectedSource;
|
|
1474
|
+
connectionId: string;
|
|
1475
|
+
accountId: string;
|
|
1476
|
+
workspaceId: string;
|
|
1477
|
+
initiatingSubjectId: string;
|
|
1478
|
+
authorization: string | undefined;
|
|
1479
|
+
fullReconciliationIntervalMs: number;
|
|
1480
|
+
observedExternalObjectIds: (ids: string[]) => Promise<Set<string>>;
|
|
1481
|
+
provider?: GoogleDriveSyncProviderPort;
|
|
1482
|
+
limits: {
|
|
1483
|
+
maxItems: number;
|
|
1484
|
+
maxBytes: number;
|
|
1485
|
+
maxFileBytes: number;
|
|
1486
|
+
maxProviderRequests: number;
|
|
1487
|
+
maxElapsedSeconds: number;
|
|
1488
|
+
};
|
|
1489
|
+
}): KnowledgeSourceSyncDriver<GoogleDriveInventoryEntry, GoogleDriveInventoryStopReason> {
|
|
1490
|
+
const maxElapsedMs = input.limits.maxElapsedSeconds * 1_000;
|
|
1491
|
+
const provider: GoogleDriveSyncProviderPort = input.provider ?? {
|
|
1492
|
+
getStartPageToken: async (driveId) =>
|
|
1493
|
+
await getDriveStartPageToken(driveId, input.authorization),
|
|
1494
|
+
listChanges: async (pageToken, driveId, pageSize) =>
|
|
1495
|
+
await listDriveChanges(pageToken, driveId, pageSize, input.authorization),
|
|
1496
|
+
getFile: async (fileId) => await getDriveFile(fileId, input.authorization),
|
|
1497
|
+
listChildren: async (request) => await listDriveChildren(request, input.authorization),
|
|
1498
|
+
};
|
|
1499
|
+
const clock = provider.now ?? Date.now;
|
|
1500
|
+
const budgetLimits = {
|
|
1501
|
+
maxItems: input.limits.maxItems,
|
|
1502
|
+
maxProviderRequests: input.limits.maxProviderRequests,
|
|
1503
|
+
maxElapsedMs,
|
|
1504
|
+
};
|
|
1505
|
+
return {
|
|
1506
|
+
providerKey: "google_drive",
|
|
1507
|
+
providerDomain: GOOGLE_DRIVE_PROVIDER_DOMAIN,
|
|
1508
|
+
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
1509
|
+
inventory: async (executionCheckpoint, providerCursor) => {
|
|
1510
|
+
const expectedCursor = {
|
|
1511
|
+
connectionId: input.connectionId,
|
|
1512
|
+
googlePermissionId: input.metadata.googlePermissionId,
|
|
1513
|
+
sourceId: input.selectedSource.id,
|
|
1514
|
+
driveId: input.selectedSource.driveId,
|
|
1515
|
+
};
|
|
1516
|
+
const legacyFullCheckpoint = isLegacyGoogleDriveFullReconciliationCheckpoint(
|
|
1517
|
+
executionCheckpoint,
|
|
1518
|
+
expectedCursor,
|
|
1519
|
+
);
|
|
1520
|
+
const cursor = parseGoogleDriveChangesCursor(providerCursor, expectedCursor);
|
|
1521
|
+
const checkpoint = parseGoogleDriveExecutionCheckpoint(
|
|
1522
|
+
executionCheckpoint,
|
|
1523
|
+
expectedCursor,
|
|
1524
|
+
budgetLimits,
|
|
1525
|
+
);
|
|
1526
|
+
const now = new Date(clock());
|
|
1527
|
+
|
|
1528
|
+
const runFullReconciliation = async (full: {
|
|
1529
|
+
boundaryId: string;
|
|
1530
|
+
startPageToken: string;
|
|
1531
|
+
cursorInvalidated: boolean;
|
|
1532
|
+
budgetBeforeInventory: GoogleDriveSyncBudget;
|
|
1533
|
+
inventoryElapsedMs: number;
|
|
1534
|
+
inventoryCheckpoint: GoogleDriveInventoryCheckpoint | null;
|
|
1535
|
+
revisionFloors: GoogleDriveRevisionFloor[];
|
|
1536
|
+
}) => {
|
|
1537
|
+
const inventoryItems = full.inventoryCheckpoint?.totals.itemCount ?? 0;
|
|
1538
|
+
const inventoryProviderRequests = full.inventoryCheckpoint?.totals.apiRequestCount ?? 0;
|
|
1539
|
+
const budgetBeforeInvocation = {
|
|
1540
|
+
examinedItems: full.budgetBeforeInventory.examinedItems + inventoryItems,
|
|
1541
|
+
providerRequests: full.budgetBeforeInventory.providerRequests + inventoryProviderRequests,
|
|
1542
|
+
elapsedMs: full.budgetBeforeInventory.elapsedMs + full.inventoryElapsedMs,
|
|
1543
|
+
} satisfies GoogleDriveSyncBudget;
|
|
1544
|
+
const hardStopReason = googleDriveHardLimitReason(budgetBeforeInvocation, budgetLimits);
|
|
1545
|
+
const fullCheckpoint = buildGoogleDriveFullReconciliationCheckpoint({
|
|
1546
|
+
...expectedCursor,
|
|
1547
|
+
...full,
|
|
1548
|
+
});
|
|
1549
|
+
if (hardStopReason) {
|
|
1550
|
+
return {
|
|
1551
|
+
status: "paused" as const,
|
|
1552
|
+
stopReason: hardStopReason,
|
|
1553
|
+
entries: [],
|
|
1554
|
+
checkpoint: fullCheckpoint,
|
|
1555
|
+
providerCursor,
|
|
1556
|
+
authoritativeFullScan: false,
|
|
1557
|
+
cursorInvalidated: full.cursorInvalidated,
|
|
1558
|
+
providerRequests: budgetBeforeInvocation.providerRequests,
|
|
1559
|
+
elapsedMs: budgetBeforeInvocation.elapsedMs,
|
|
1560
|
+
hardLimitReached: true,
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
const phaseMaxItems = input.limits.maxItems - full.budgetBeforeInventory.examinedItems;
|
|
1565
|
+
const phaseMaxProviderRequests =
|
|
1566
|
+
input.limits.maxProviderRequests - full.budgetBeforeInventory.providerRequests;
|
|
1567
|
+
const remainingElapsedMs = maxElapsedMs - budgetBeforeInvocation.elapsedMs;
|
|
1568
|
+
const inventory = await inventoryGoogleDriveSource({
|
|
1569
|
+
googlePermissionId: input.metadata.googlePermissionId,
|
|
1570
|
+
googleEmail: input.metadata.googleEmail,
|
|
1571
|
+
source: input.selectedSource,
|
|
1572
|
+
accountId: input.accountId,
|
|
1573
|
+
workspaceId: input.workspaceId,
|
|
1574
|
+
connectionSubjectId: input.initiatingSubjectId,
|
|
1575
|
+
limits: {
|
|
1576
|
+
maxItems: phaseMaxItems,
|
|
1577
|
+
maxKnownBytes: input.limits.maxBytes,
|
|
1578
|
+
maxApiRequests: phaseMaxProviderRequests,
|
|
1579
|
+
maxElapsedMs: Math.min(GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS, remainingElapsedMs),
|
|
1580
|
+
maxFileBytes: input.limits.maxFileBytes,
|
|
1581
|
+
maxFolders: Math.min(phaseMaxItems * 2, 2_000),
|
|
1582
|
+
pageSize: Math.min(phaseMaxItems, 100),
|
|
1583
|
+
},
|
|
1584
|
+
checkpoint: full.inventoryCheckpoint,
|
|
1585
|
+
listChildren: provider.listChildren,
|
|
1586
|
+
now: clock,
|
|
1587
|
+
});
|
|
1588
|
+
const inventoryElapsedMs = full.inventoryElapsedMs + inventory.run.elapsedMs;
|
|
1589
|
+
const budget = {
|
|
1590
|
+
examinedItems: full.budgetBeforeInventory.examinedItems + inventory.totals.itemCount,
|
|
1591
|
+
providerRequests:
|
|
1592
|
+
full.budgetBeforeInventory.providerRequests + inventory.totals.apiRequestCount,
|
|
1593
|
+
elapsedMs: full.budgetBeforeInventory.elapsedMs + inventoryElapsedMs,
|
|
1594
|
+
} satisfies GoogleDriveSyncBudget;
|
|
1595
|
+
const reconciled = reconcileGoogleDriveRevisionFloors(
|
|
1596
|
+
inventory.entries,
|
|
1597
|
+
full.revisionFloors,
|
|
1598
|
+
input.limits.maxItems,
|
|
1599
|
+
);
|
|
1600
|
+
if (reconciled.conflict) {
|
|
1601
|
+
return {
|
|
1602
|
+
status: "paused" as const,
|
|
1603
|
+
stopReason: "incomplete_search" as const,
|
|
1604
|
+
entries: [],
|
|
1605
|
+
checkpoint: fullCheckpoint,
|
|
1606
|
+
providerCursor,
|
|
1607
|
+
authoritativeFullScan: false,
|
|
1608
|
+
cursorInvalidated: full.cursorInvalidated,
|
|
1609
|
+
providerRequests: budget.providerRequests,
|
|
1610
|
+
elapsedMs: budget.elapsedMs,
|
|
1611
|
+
hardLimitReached: false,
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
const nextCursor =
|
|
1615
|
+
inventory.status === "complete"
|
|
1616
|
+
? buildGoogleDriveChangesCursor({
|
|
1617
|
+
...expectedCursor,
|
|
1618
|
+
boundaryId: full.boundaryId,
|
|
1619
|
+
pageToken: full.startPageToken,
|
|
1620
|
+
previousGeneration: cursor?.cursorGeneration ?? 0,
|
|
1621
|
+
reconciledAt: now,
|
|
1622
|
+
fullReconciliationIntervalMs: input.fullReconciliationIntervalMs,
|
|
1623
|
+
})
|
|
1624
|
+
: providerCursor;
|
|
1625
|
+
return {
|
|
1626
|
+
status: inventory.status,
|
|
1627
|
+
stopReason: inventory.stopReason,
|
|
1628
|
+
entries: reconciled.entries,
|
|
1629
|
+
checkpoint:
|
|
1630
|
+
inventory.status === "paused" && inventory.checkpoint
|
|
1631
|
+
? buildGoogleDriveFullReconciliationCheckpoint({
|
|
1632
|
+
...expectedCursor,
|
|
1633
|
+
boundaryId: full.boundaryId,
|
|
1634
|
+
startPageToken: full.startPageToken,
|
|
1635
|
+
cursorInvalidated: full.cursorInvalidated,
|
|
1636
|
+
budgetBeforeInventory: full.budgetBeforeInventory,
|
|
1637
|
+
inventoryElapsedMs,
|
|
1638
|
+
inventoryCheckpoint: inventory.checkpoint,
|
|
1639
|
+
revisionFloors: reconciled.revisionFloors,
|
|
1640
|
+
})
|
|
1641
|
+
: null,
|
|
1642
|
+
providerCursor: nextCursor,
|
|
1643
|
+
authoritativeFullScan: inventory.status === "complete",
|
|
1644
|
+
cursorInvalidated: full.cursorInvalidated,
|
|
1645
|
+
providerRequests: budget.providerRequests,
|
|
1646
|
+
elapsedMs: budget.elapsedMs,
|
|
1647
|
+
hardLimitReached:
|
|
1648
|
+
inventory.status === "paused" &&
|
|
1649
|
+
googleDriveInventoryStopIsHard(inventory.stopReason, budget, budgetLimits),
|
|
1650
|
+
};
|
|
1651
|
+
};
|
|
1652
|
+
|
|
1653
|
+
const acquireFullReconciliation = async (details: {
|
|
1654
|
+
budget: GoogleDriveSyncBudget;
|
|
1655
|
+
boundaryId: string | null;
|
|
1656
|
+
cursorInvalidated: boolean;
|
|
1657
|
+
}) => {
|
|
1658
|
+
const acquisitionStartedAt = clock();
|
|
1659
|
+
const initialElapsedMs = details.budget.elapsedMs;
|
|
1660
|
+
const budget = { ...details.budget };
|
|
1661
|
+
const refreshElapsed = () => {
|
|
1662
|
+
budget.elapsedMs = initialElapsedMs + Math.max(0, clock() - acquisitionStartedAt);
|
|
1663
|
+
};
|
|
1664
|
+
const requireProviderCapacity = () => {
|
|
1665
|
+
refreshElapsed();
|
|
1666
|
+
if (googleDriveHardLimitReason(budget, budgetLimits)) {
|
|
1667
|
+
throw new SyncFailure("resource_limit", false);
|
|
1668
|
+
}
|
|
1669
|
+
budget.providerRequests += 1;
|
|
1670
|
+
};
|
|
1671
|
+
|
|
1672
|
+
requireProviderCapacity();
|
|
1673
|
+
const startPageToken = await provider.getStartPageToken(input.selectedSource.driveId);
|
|
1674
|
+
refreshElapsed();
|
|
1675
|
+
|
|
1676
|
+
let boundaryId = details.boundaryId;
|
|
1677
|
+
if (!boundaryId) {
|
|
1678
|
+
if (input.selectedSource.id === "root" && input.selectedSource.driveId === null) {
|
|
1679
|
+
requireProviderCapacity();
|
|
1680
|
+
try {
|
|
1681
|
+
boundaryId = await resolveGoogleDriveChangesBoundaryId({
|
|
1682
|
+
source: input.selectedSource,
|
|
1683
|
+
getFile: provider.getFile,
|
|
1684
|
+
});
|
|
1685
|
+
} catch (error) {
|
|
1686
|
+
if (error instanceof GoogleDriveChangesProtocolError) {
|
|
1687
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
1688
|
+
}
|
|
1689
|
+
throw error;
|
|
1690
|
+
}
|
|
1691
|
+
refreshElapsed();
|
|
1692
|
+
} else {
|
|
1693
|
+
boundaryId = input.selectedSource.id;
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
return {
|
|
1698
|
+
boundaryId,
|
|
1699
|
+
startPageToken,
|
|
1700
|
+
cursorInvalidated: details.cursorInvalidated,
|
|
1701
|
+
budgetBeforeInventory: budget,
|
|
1702
|
+
inventoryElapsedMs: 0,
|
|
1703
|
+
inventoryCheckpoint: null,
|
|
1704
|
+
revisionFloors: [],
|
|
1705
|
+
};
|
|
1706
|
+
};
|
|
1707
|
+
|
|
1708
|
+
if (checkpoint?.kind === "google_drive_full_reconciliation") {
|
|
1709
|
+
return await runFullReconciliation(checkpoint);
|
|
1710
|
+
}
|
|
1711
|
+
if (!cursor) {
|
|
1712
|
+
const budget =
|
|
1713
|
+
checkpoint?.kind === "google_drive_changes"
|
|
1714
|
+
? checkpoint.changesCheckpoint.budget
|
|
1715
|
+
: emptyGoogleDriveSyncBudget();
|
|
1716
|
+
return await runFullReconciliation(
|
|
1717
|
+
await acquireFullReconciliation({
|
|
1718
|
+
budget,
|
|
1719
|
+
boundaryId: null,
|
|
1720
|
+
cursorInvalidated: providerCursor !== null,
|
|
1721
|
+
}),
|
|
1722
|
+
);
|
|
1723
|
+
}
|
|
1724
|
+
if (!checkpoint && !legacyFullCheckpoint && googleDriveFullReconciliationDue(cursor, now)) {
|
|
1725
|
+
return await runFullReconciliation(
|
|
1726
|
+
await acquireFullReconciliation({
|
|
1727
|
+
budget: emptyGoogleDriveSyncBudget(),
|
|
1728
|
+
boundaryId: cursor.boundaryId,
|
|
1729
|
+
cursorInvalidated: false,
|
|
1730
|
+
}),
|
|
1731
|
+
);
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
try {
|
|
1735
|
+
const forceFullReconciliation =
|
|
1736
|
+
legacyFullCheckpoint || googleDriveFullReconciliationDue(cursor, now);
|
|
1737
|
+
const changesCheckpoint =
|
|
1738
|
+
checkpoint?.kind === "google_drive_changes"
|
|
1739
|
+
? {
|
|
1740
|
+
...checkpoint.changesCheckpoint,
|
|
1741
|
+
requiresFullReconciliation:
|
|
1742
|
+
checkpoint.changesCheckpoint.requiresFullReconciliation ||
|
|
1743
|
+
forceFullReconciliation,
|
|
1744
|
+
}
|
|
1745
|
+
: null;
|
|
1746
|
+
const changes = await drainGoogleDriveChanges({
|
|
1747
|
+
source: input.selectedSource,
|
|
1748
|
+
cursor,
|
|
1749
|
+
checkpoint: changesCheckpoint,
|
|
1750
|
+
maxItems: input.limits.maxItems,
|
|
1751
|
+
maxProviderRequests: input.limits.maxProviderRequests,
|
|
1752
|
+
maxElapsedMs,
|
|
1753
|
+
maxInvocationElapsedMs: Math.min(GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS, maxElapsedMs),
|
|
1754
|
+
maxFileBytes: input.limits.maxFileBytes,
|
|
1755
|
+
listChanges: async (pageToken, pageSize) =>
|
|
1756
|
+
await provider.listChanges(pageToken, input.selectedSource.driveId, pageSize),
|
|
1757
|
+
getFile: provider.getFile,
|
|
1758
|
+
observedExternalObjectIds: input.observedExternalObjectIds,
|
|
1759
|
+
now: clock,
|
|
1760
|
+
});
|
|
1761
|
+
if (changes.status === "paused" && changes.checkpoint) {
|
|
1762
|
+
return {
|
|
1763
|
+
status: "paused" as const,
|
|
1764
|
+
stopReason: changes.stopReason,
|
|
1765
|
+
entries: changes.entries,
|
|
1766
|
+
checkpoint: {
|
|
1767
|
+
version: 2,
|
|
1768
|
+
kind: "google_drive_changes",
|
|
1769
|
+
...expectedCursor,
|
|
1770
|
+
changesCheckpoint: {
|
|
1771
|
+
...changes.checkpoint,
|
|
1772
|
+
requiresFullReconciliation:
|
|
1773
|
+
changes.checkpoint.requiresFullReconciliation || forceFullReconciliation,
|
|
1774
|
+
},
|
|
1775
|
+
} satisfies GoogleDriveExecutionCheckpoint,
|
|
1776
|
+
providerCursor,
|
|
1777
|
+
authoritativeFullScan: false,
|
|
1778
|
+
cursorInvalidated: false,
|
|
1779
|
+
providerRequests: changes.providerRequests,
|
|
1780
|
+
elapsedMs: changes.elapsedMs,
|
|
1781
|
+
hardLimitReached: changes.hardLimitReached,
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
if (!changes.newStartPageToken) throw new Error("google_drive_changes_cursor_missing");
|
|
1785
|
+
if (changes.requiresFullReconciliation || forceFullReconciliation) {
|
|
1786
|
+
const hardStopReason = googleDriveHardLimitReason(changes.budget, budgetLimits);
|
|
1787
|
+
return {
|
|
1788
|
+
status: "paused" as const,
|
|
1789
|
+
stopReason: hardStopReason ?? ("elapsed_time_limit" as const),
|
|
1790
|
+
entries: changes.entries,
|
|
1791
|
+
checkpoint: buildGoogleDriveFullReconciliationCheckpoint({
|
|
1792
|
+
...expectedCursor,
|
|
1793
|
+
boundaryId: cursor.boundaryId,
|
|
1794
|
+
startPageToken: changes.newStartPageToken,
|
|
1795
|
+
cursorInvalidated: false,
|
|
1796
|
+
budgetBeforeInventory: changes.budget,
|
|
1797
|
+
inventoryElapsedMs: 0,
|
|
1798
|
+
inventoryCheckpoint: null,
|
|
1799
|
+
revisionFloors: revisionFloorsForEntries(changes.entries, input.limits.maxItems),
|
|
1800
|
+
}),
|
|
1801
|
+
providerCursor,
|
|
1802
|
+
authoritativeFullScan: false,
|
|
1803
|
+
cursorInvalidated: false,
|
|
1804
|
+
providerRequests: changes.providerRequests,
|
|
1805
|
+
elapsedMs: changes.elapsedMs,
|
|
1806
|
+
hardLimitReached: hardStopReason !== null,
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1809
|
+
return {
|
|
1810
|
+
status: "complete" as const,
|
|
1811
|
+
stopReason: null,
|
|
1812
|
+
entries: changes.entries,
|
|
1813
|
+
checkpoint: null,
|
|
1814
|
+
providerCursor: advanceGoogleDriveChangesCursor(cursor, changes.newStartPageToken),
|
|
1815
|
+
authoritativeFullScan: false,
|
|
1816
|
+
cursorInvalidated: false,
|
|
1817
|
+
providerRequests: changes.providerRequests,
|
|
1818
|
+
elapsedMs: changes.elapsedMs,
|
|
1819
|
+
hardLimitReached: false,
|
|
1820
|
+
};
|
|
1821
|
+
} catch (error) {
|
|
1822
|
+
if (error instanceof GoogleDriveChangesProtocolError) {
|
|
1823
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
1824
|
+
}
|
|
1825
|
+
if (!(error instanceof GoogleDriveCursorInvalidError)) throw error;
|
|
1826
|
+
const full = await acquireFullReconciliation({
|
|
1827
|
+
budget: error.budget ?? emptyGoogleDriveSyncBudget(),
|
|
1828
|
+
boundaryId: cursor.boundaryId,
|
|
1829
|
+
cursorInvalidated: true,
|
|
1830
|
+
});
|
|
1831
|
+
const hardStopReason = googleDriveHardLimitReason(full.budgetBeforeInventory, budgetLimits);
|
|
1832
|
+
return {
|
|
1833
|
+
status: "paused" as const,
|
|
1834
|
+
stopReason: hardStopReason ?? ("elapsed_time_limit" as const),
|
|
1835
|
+
entries: [],
|
|
1836
|
+
checkpoint: buildGoogleDriveFullReconciliationCheckpoint({
|
|
1837
|
+
...expectedCursor,
|
|
1838
|
+
...full,
|
|
1839
|
+
}),
|
|
1840
|
+
providerCursor,
|
|
1841
|
+
authoritativeFullScan: false,
|
|
1842
|
+
cursorInvalidated: true,
|
|
1843
|
+
providerRequests: full.budgetBeforeInventory.providerRequests,
|
|
1844
|
+
elapsedMs: full.budgetBeforeInventory.elapsedMs,
|
|
1845
|
+
hardLimitReached: hardStopReason !== null,
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
},
|
|
1849
|
+
fetchContent: async (entry, maxBytes) =>
|
|
1850
|
+
await fetchDriveBytes(entry, input.authorization, maxBytes),
|
|
1851
|
+
citationLocator: (entry) => ({
|
|
1852
|
+
version: 1,
|
|
1853
|
+
providerKey: "google-drive",
|
|
1854
|
+
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
1855
|
+
externalObjectId: entry.externalObjectId,
|
|
1856
|
+
sourceUri: entry.sourceUri,
|
|
1857
|
+
}),
|
|
1858
|
+
};
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
type GoogleDriveCheckpointIdentity = {
|
|
1862
|
+
connectionId: string;
|
|
1863
|
+
googlePermissionId: string;
|
|
1864
|
+
sourceId: string;
|
|
1865
|
+
driveId: string | null;
|
|
1866
|
+
};
|
|
1867
|
+
|
|
1868
|
+
type GoogleDriveRevisionFloor = [externalObjectId: string, providerRevision: string | null];
|
|
1869
|
+
|
|
1870
|
+
type GoogleDriveFullReconciliationCheckpoint = GoogleDriveCheckpointIdentity & {
|
|
1871
|
+
version: 3;
|
|
1872
|
+
kind: "google_drive_full_reconciliation";
|
|
1873
|
+
boundaryId: string;
|
|
1874
|
+
startPageToken: string;
|
|
1875
|
+
cursorInvalidated: boolean;
|
|
1876
|
+
budgetBeforeInventory: GoogleDriveSyncBudget;
|
|
1877
|
+
inventoryElapsedMs: number;
|
|
1878
|
+
inventoryCheckpoint: GoogleDriveInventoryCheckpoint | null;
|
|
1879
|
+
revisionFloors: GoogleDriveRevisionFloor[];
|
|
1880
|
+
};
|
|
1881
|
+
|
|
1882
|
+
type GoogleDriveExecutionCheckpoint = GoogleDriveCheckpointIdentity &
|
|
1883
|
+
(
|
|
1884
|
+
| {
|
|
1885
|
+
version: 2;
|
|
1886
|
+
kind: "google_drive_changes";
|
|
1887
|
+
changesCheckpoint: GoogleDriveChangesCheckpoint;
|
|
1888
|
+
}
|
|
1889
|
+
| GoogleDriveFullReconciliationCheckpoint
|
|
1890
|
+
);
|
|
1891
|
+
|
|
1892
|
+
function isLegacyGoogleDriveFullReconciliationCheckpoint(
|
|
1893
|
+
value: Record<string, unknown> | null,
|
|
1894
|
+
expected: GoogleDriveCheckpointIdentity,
|
|
1895
|
+
): boolean {
|
|
1896
|
+
return (
|
|
1897
|
+
value?.version === 2 &&
|
|
1898
|
+
value.kind === "google_drive_full_reconciliation" &&
|
|
1899
|
+
value.connectionId === expected.connectionId &&
|
|
1900
|
+
value.googlePermissionId === expected.googlePermissionId &&
|
|
1901
|
+
value.sourceId === expected.sourceId &&
|
|
1902
|
+
value.driveId === expected.driveId &&
|
|
1903
|
+
encodedJsonBytes(value) <= GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES
|
|
1904
|
+
);
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
function parseGoogleDriveExecutionCheckpoint(
|
|
1908
|
+
value: Record<string, unknown> | null,
|
|
1909
|
+
expected: GoogleDriveCheckpointIdentity,
|
|
1910
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
1911
|
+
): GoogleDriveExecutionCheckpoint | null {
|
|
1912
|
+
if (
|
|
1913
|
+
!value ||
|
|
1914
|
+
value.connectionId !== expected.connectionId ||
|
|
1915
|
+
value.googlePermissionId !== expected.googlePermissionId ||
|
|
1916
|
+
value.sourceId !== expected.sourceId ||
|
|
1917
|
+
value.driveId !== expected.driveId ||
|
|
1918
|
+
encodedJsonBytes(value) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES
|
|
1919
|
+
) {
|
|
1920
|
+
return null;
|
|
1921
|
+
}
|
|
1922
|
+
if (value.version === 2 && value.kind === "google_drive_changes") {
|
|
1923
|
+
const changesCheckpoint = parseGoogleDriveChangesCheckpoint(value.changesCheckpoint, limits);
|
|
1924
|
+
return changesCheckpoint
|
|
1925
|
+
? ({ ...value, changesCheckpoint } as GoogleDriveExecutionCheckpoint)
|
|
1926
|
+
: null;
|
|
1927
|
+
}
|
|
1928
|
+
if (value.version === 3 && value.kind === "google_drive_full_reconciliation") {
|
|
1929
|
+
const budgetBeforeInventory = parseGoogleDriveSyncBudget(value.budgetBeforeInventory, limits);
|
|
1930
|
+
const revisionFloors = parseGoogleDriveRevisionFloors(value.revisionFloors, limits.maxItems);
|
|
1931
|
+
if (
|
|
1932
|
+
typeof value.boundaryId !== "string" ||
|
|
1933
|
+
value.boundaryId.length < 1 ||
|
|
1934
|
+
value.boundaryId.length > 1024 ||
|
|
1935
|
+
typeof value.startPageToken !== "string" ||
|
|
1936
|
+
value.startPageToken.length < 1 ||
|
|
1937
|
+
value.startPageToken.length > 4096 ||
|
|
1938
|
+
typeof value.cursorInvalidated !== "boolean" ||
|
|
1939
|
+
!budgetBeforeInventory ||
|
|
1940
|
+
!revisionFloors ||
|
|
1941
|
+
!Number.isSafeInteger(value.inventoryElapsedMs) ||
|
|
1942
|
+
Number(value.inventoryElapsedMs) < 0 ||
|
|
1943
|
+
Number(value.inventoryElapsedMs) > limits.maxElapsedMs ||
|
|
1944
|
+
budgetBeforeInventory.elapsedMs + Number(value.inventoryElapsedMs) > limits.maxElapsedMs ||
|
|
1945
|
+
(value.inventoryCheckpoint !== null &&
|
|
1946
|
+
(!value.inventoryCheckpoint ||
|
|
1947
|
+
typeof value.inventoryCheckpoint !== "object" ||
|
|
1948
|
+
Array.isArray(value.inventoryCheckpoint)))
|
|
1949
|
+
) {
|
|
1950
|
+
return null;
|
|
1951
|
+
}
|
|
1952
|
+
return {
|
|
1953
|
+
...value,
|
|
1954
|
+
budgetBeforeInventory,
|
|
1955
|
+
inventoryElapsedMs: Number(value.inventoryElapsedMs),
|
|
1956
|
+
revisionFloors,
|
|
1957
|
+
} as GoogleDriveExecutionCheckpoint;
|
|
1958
|
+
}
|
|
1959
|
+
return null;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
function buildGoogleDriveFullReconciliationCheckpoint(
|
|
1963
|
+
input: Omit<GoogleDriveFullReconciliationCheckpoint, "version" | "kind">,
|
|
1964
|
+
): GoogleDriveFullReconciliationCheckpoint {
|
|
1965
|
+
const checkpoint = {
|
|
1966
|
+
version: 3,
|
|
1967
|
+
kind: "google_drive_full_reconciliation",
|
|
1968
|
+
...input,
|
|
1969
|
+
revisionFloors: input.revisionFloors.map((floor) => [...floor]),
|
|
1970
|
+
} satisfies GoogleDriveFullReconciliationCheckpoint;
|
|
1971
|
+
if (encodedJsonBytes(checkpoint) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES) {
|
|
1972
|
+
throw new SyncFailure("resource_limit", false);
|
|
1973
|
+
}
|
|
1974
|
+
return checkpoint;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
function parseGoogleDriveRevisionFloors(
|
|
1978
|
+
value: unknown,
|
|
1979
|
+
maxItems: number,
|
|
1980
|
+
): GoogleDriveRevisionFloor[] | null {
|
|
1981
|
+
if (!Array.isArray(value) || value.length > maxItems) return null;
|
|
1982
|
+
const floors: GoogleDriveRevisionFloor[] = [];
|
|
1983
|
+
const seen = new Set<string>();
|
|
1984
|
+
for (const floor of value) {
|
|
1985
|
+
if (
|
|
1986
|
+
!Array.isArray(floor) ||
|
|
1987
|
+
floor.length !== 2 ||
|
|
1988
|
+
typeof floor[0] !== "string" ||
|
|
1989
|
+
floor[0].trim().length < 1 ||
|
|
1990
|
+
floor[0].length > 1024 ||
|
|
1991
|
+
(floor[1] !== null &&
|
|
1992
|
+
(typeof floor[1] !== "string" || floor[1].length < 1 || floor[1].length > 256)) ||
|
|
1993
|
+
seen.has(floor[0])
|
|
1994
|
+
) {
|
|
1995
|
+
return null;
|
|
1996
|
+
}
|
|
1997
|
+
seen.add(floor[0]);
|
|
1998
|
+
floors.push([floor[0], floor[1]]);
|
|
1999
|
+
}
|
|
2000
|
+
return floors;
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
function revisionFloorsForEntries(
|
|
2004
|
+
entries: GoogleDriveInventoryEntry[],
|
|
2005
|
+
maxItems: number,
|
|
2006
|
+
): GoogleDriveRevisionFloor[] {
|
|
2007
|
+
const reconciled = reconcileGoogleDriveRevisionFloors(entries, [], maxItems);
|
|
2008
|
+
if (reconciled.conflict) throw new SyncFailure("provider_payload_invalid", false);
|
|
2009
|
+
return reconciled.revisionFloors;
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
export function shouldProcessGoogleDriveDurableObservation(input: {
|
|
2013
|
+
entry: Pick<GoogleDriveInventoryEntry, "externalObjectId">;
|
|
2014
|
+
observation: KnowledgeSourceSyncObjectObservationResult;
|
|
2015
|
+
sourceLifecycleGeneration: number;
|
|
2016
|
+
aclGeneration: number;
|
|
2017
|
+
}): boolean {
|
|
2018
|
+
if (input.observation.externalObjectId !== input.entry.externalObjectId) {
|
|
2019
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2020
|
+
}
|
|
2021
|
+
if (
|
|
2022
|
+
input.observation.disposition === "accepted" ||
|
|
2023
|
+
input.observation.disposition === "replayed"
|
|
2024
|
+
) {
|
|
2025
|
+
return true;
|
|
2026
|
+
}
|
|
2027
|
+
if (input.observation.disposition === "conflict") {
|
|
2028
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2029
|
+
}
|
|
2030
|
+
const current = input.observation.currentVersion;
|
|
2031
|
+
const comparison = current
|
|
2032
|
+
? compareCanonicalDecimalProviderRevisions(
|
|
2033
|
+
current.providerRevision,
|
|
2034
|
+
input.observation.providerRevision,
|
|
2035
|
+
)
|
|
2036
|
+
: null;
|
|
2037
|
+
const fullyProcessed =
|
|
2038
|
+
current !== null &&
|
|
2039
|
+
current.objectLifecycleState === "active" &&
|
|
2040
|
+
current.sourceLifecycleGeneration === input.sourceLifecycleGeneration &&
|
|
2041
|
+
current.objectLifecycleGeneration === current.currentObjectLifecycleGeneration &&
|
|
2042
|
+
current.aclGeneration === input.aclGeneration &&
|
|
2043
|
+
current.indexObligationStatus === "indexed" &&
|
|
2044
|
+
comparison !== null &&
|
|
2045
|
+
comparison >= 0 &&
|
|
2046
|
+
(comparison > 0 || current.metadataHash === input.observation.metadataHash);
|
|
2047
|
+
if (!fullyProcessed) throw new SyncFailure("provider_payload_invalid", false);
|
|
2048
|
+
return false;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
export function mergeGoogleDriveDurableObservationFloors(
|
|
2052
|
+
checkpoint: Record<string, unknown> | null,
|
|
2053
|
+
observations: KnowledgeSourceSyncObservationFloor[],
|
|
2054
|
+
maxItems: number,
|
|
2055
|
+
): Record<string, unknown> | null {
|
|
2056
|
+
if (
|
|
2057
|
+
!checkpoint ||
|
|
2058
|
+
checkpoint.version !== 3 ||
|
|
2059
|
+
checkpoint.kind !== "google_drive_full_reconciliation"
|
|
2060
|
+
) {
|
|
2061
|
+
return checkpoint;
|
|
2062
|
+
}
|
|
2063
|
+
const parsedFloors = parseGoogleDriveRevisionFloors(checkpoint.revisionFloors, maxItems);
|
|
2064
|
+
if (!parsedFloors) throw new SyncFailure("provider_payload_invalid", false);
|
|
2065
|
+
const floors = new Map<string, string | null>(parsedFloors);
|
|
2066
|
+
for (const observation of observations) {
|
|
2067
|
+
floors.set(observation.externalObjectId, observation.providerRevision);
|
|
2068
|
+
}
|
|
2069
|
+
if (floors.size > maxItems) throw new SyncFailure("resource_limit", false);
|
|
2070
|
+
const merged = {
|
|
2071
|
+
...checkpoint,
|
|
2072
|
+
revisionFloors: [...floors],
|
|
2073
|
+
};
|
|
2074
|
+
if (encodedJsonBytes(merged) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES) {
|
|
2075
|
+
throw new SyncFailure("resource_limit", false);
|
|
2076
|
+
}
|
|
2077
|
+
return merged;
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
function reconcileGoogleDriveRevisionFloors(
|
|
2081
|
+
entries: GoogleDriveInventoryEntry[],
|
|
2082
|
+
existingFloors: GoogleDriveRevisionFloor[],
|
|
2083
|
+
maxItems: number,
|
|
2084
|
+
): {
|
|
2085
|
+
entries: GoogleDriveInventoryEntry[];
|
|
2086
|
+
revisionFloors: GoogleDriveRevisionFloor[];
|
|
2087
|
+
conflict: boolean;
|
|
2088
|
+
} {
|
|
2089
|
+
const floors = new Map<string, string | null>(existingFloors);
|
|
2090
|
+
const accepted: GoogleDriveInventoryEntry[] = [];
|
|
2091
|
+
for (const entry of entries) {
|
|
2092
|
+
if (!floors.has(entry.externalObjectId)) {
|
|
2093
|
+
floors.set(entry.externalObjectId, entry.externalVersionId);
|
|
2094
|
+
accepted.push(entry);
|
|
2095
|
+
} else {
|
|
2096
|
+
const comparison = compareCanonicalDecimalProviderRevisions(
|
|
2097
|
+
entry.externalVersionId,
|
|
2098
|
+
floors.get(entry.externalObjectId) ?? null,
|
|
2099
|
+
);
|
|
2100
|
+
if (comparison === null) {
|
|
2101
|
+
return {
|
|
2102
|
+
entries: [],
|
|
2103
|
+
revisionFloors: [...floors],
|
|
2104
|
+
conflict: true,
|
|
2105
|
+
};
|
|
2106
|
+
}
|
|
2107
|
+
if (comparison > 0) {
|
|
2108
|
+
floors.set(entry.externalObjectId, entry.externalVersionId);
|
|
2109
|
+
accepted.push(entry);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
if (floors.size > maxItems) throw new SyncFailure("resource_limit", false);
|
|
2113
|
+
}
|
|
2114
|
+
return {
|
|
2115
|
+
entries: accepted,
|
|
2116
|
+
revisionFloors: [...floors],
|
|
2117
|
+
conflict: false,
|
|
2118
|
+
};
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
function encodedJsonBytes(value: unknown): number {
|
|
2122
|
+
try {
|
|
2123
|
+
return new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
2124
|
+
} catch {
|
|
2125
|
+
return Number.POSITIVE_INFINITY;
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
function parseGoogleDriveSyncBudget(
|
|
2130
|
+
value: unknown,
|
|
2131
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
2132
|
+
): GoogleDriveSyncBudget | null {
|
|
2133
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
2134
|
+
const row = value as Record<string, unknown>;
|
|
2135
|
+
for (const [key, maximum] of [
|
|
2136
|
+
["examinedItems", limits.maxItems],
|
|
2137
|
+
["providerRequests", limits.maxProviderRequests],
|
|
2138
|
+
["elapsedMs", limits.maxElapsedMs],
|
|
2139
|
+
] as const) {
|
|
2140
|
+
const budgetValue = row[key];
|
|
2141
|
+
if (
|
|
2142
|
+
!Number.isSafeInteger(budgetValue) ||
|
|
2143
|
+
Number(budgetValue) < 0 ||
|
|
2144
|
+
Number(budgetValue) > maximum
|
|
2145
|
+
) {
|
|
2146
|
+
return null;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
return value as GoogleDriveSyncBudget;
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
function emptyGoogleDriveSyncBudget(): GoogleDriveSyncBudget {
|
|
2153
|
+
return { examinedItems: 0, providerRequests: 0, elapsedMs: 0 };
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
function googleDriveHardLimitReason(
|
|
2157
|
+
budget: GoogleDriveSyncBudget,
|
|
2158
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
2159
|
+
): Extract<
|
|
2160
|
+
GoogleDriveInventoryStopReason,
|
|
2161
|
+
"api_request_limit" | "elapsed_time_limit" | "item_limit"
|
|
2162
|
+
> | null {
|
|
2163
|
+
if (budget.examinedItems >= limits.maxItems) return "item_limit";
|
|
2164
|
+
if (budget.providerRequests >= limits.maxProviderRequests) return "api_request_limit";
|
|
2165
|
+
if (budget.elapsedMs >= limits.maxElapsedMs) return "elapsed_time_limit";
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
function googleDriveInventoryStopIsHard(
|
|
2170
|
+
reason: GoogleDriveInventoryStopReason | null,
|
|
2171
|
+
budget: GoogleDriveSyncBudget,
|
|
2172
|
+
limits: { maxItems: number; maxProviderRequests: number; maxElapsedMs: number },
|
|
2173
|
+
): boolean {
|
|
2174
|
+
if (reason === "api_request_limit" || reason === "item_limit" || reason === "known_byte_limit") {
|
|
2175
|
+
return true;
|
|
2176
|
+
}
|
|
2177
|
+
return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
async function getDriveStartPageToken(
|
|
2181
|
+
driveId: string | null,
|
|
2182
|
+
authorization: string | undefined,
|
|
2183
|
+
): Promise<string> {
|
|
2184
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2185
|
+
const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
|
|
2186
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2187
|
+
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2188
|
+
const response = await fetch(url, {
|
|
2189
|
+
headers: { authorization, accept: "application/json" },
|
|
2190
|
+
redirect: "error",
|
|
2191
|
+
signal: AbortSignal.timeout(15_000),
|
|
2192
|
+
});
|
|
2193
|
+
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2194
|
+
const payload = (await readResponseJsonBounded(
|
|
2195
|
+
response,
|
|
2196
|
+
DRIVE_JSON_MAX_BYTES,
|
|
2197
|
+
"Google Drive start page token",
|
|
2198
|
+
)) as Record<string, unknown>;
|
|
2199
|
+
return requiredString(payload.startPageToken, "start_page_token");
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
async function listDriveChanges(
|
|
2203
|
+
pageToken: string,
|
|
2204
|
+
driveId: string | null,
|
|
2205
|
+
pageSize: number,
|
|
2206
|
+
authorization: string | undefined,
|
|
2207
|
+
): Promise<GoogleDriveChangesPage> {
|
|
2208
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2209
|
+
const url = new URL(`${DRIVE_API_BASE}/changes`);
|
|
2210
|
+
url.searchParams.set("pageToken", pageToken);
|
|
2211
|
+
url.searchParams.set("pageSize", String(pageSize));
|
|
2212
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2213
|
+
url.searchParams.set("includeItemsFromAllDrives", "true");
|
|
2214
|
+
url.searchParams.set("spaces", "drive");
|
|
2215
|
+
url.searchParams.set(
|
|
2216
|
+
"fields",
|
|
2217
|
+
"nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))",
|
|
2218
|
+
);
|
|
2219
|
+
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2220
|
+
const response = await fetch(url, {
|
|
2221
|
+
headers: { authorization, accept: "application/json" },
|
|
2222
|
+
redirect: "error",
|
|
2223
|
+
signal: AbortSignal.timeout(15_000),
|
|
2224
|
+
});
|
|
2225
|
+
if (response.status === 410) throw new GoogleDriveCursorInvalidError();
|
|
2226
|
+
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2227
|
+
const payload = (await readResponseJsonBounded(
|
|
2228
|
+
response,
|
|
2229
|
+
DRIVE_JSON_MAX_BYTES,
|
|
2230
|
+
"Google Drive changes list",
|
|
2231
|
+
)) as Record<string, unknown>;
|
|
2232
|
+
const changes = Array.isArray(payload.changes) ? payload.changes : [];
|
|
2233
|
+
return {
|
|
2234
|
+
changes: changes.map((value) => {
|
|
2235
|
+
const row = value && typeof value === "object" ? (value as Record<string, unknown>) : {};
|
|
2236
|
+
const fileId = requiredString(row.fileId, "change_file_id");
|
|
2237
|
+
return {
|
|
2238
|
+
fileId,
|
|
2239
|
+
removed: row.removed === true,
|
|
2240
|
+
file: row.file && typeof row.file === "object" ? parseDriveProviderItem(row.file) : null,
|
|
2241
|
+
};
|
|
2242
|
+
}),
|
|
2243
|
+
nextPageToken: optionalString(payload.nextPageToken),
|
|
2244
|
+
newStartPageToken: optionalString(payload.newStartPageToken),
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
async function getDriveFile(
|
|
2249
|
+
fileId: string,
|
|
2250
|
+
authorization: string | undefined,
|
|
2251
|
+
): Promise<GoogleDriveInventoryProviderItem | null> {
|
|
2252
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2253
|
+
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
|
|
2254
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2255
|
+
url.searchParams.set(
|
|
2256
|
+
"fields",
|
|
2257
|
+
"id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed",
|
|
2258
|
+
);
|
|
2259
|
+
const response = await fetch(url, {
|
|
2260
|
+
headers: { authorization, accept: "application/json" },
|
|
2261
|
+
redirect: "error",
|
|
2262
|
+
signal: AbortSignal.timeout(15_000),
|
|
2263
|
+
});
|
|
2264
|
+
if (response.status === 404) return null;
|
|
2265
|
+
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2266
|
+
return parseDriveProviderItem(
|
|
2267
|
+
await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata"),
|
|
2268
|
+
);
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
function throwGoogleDriveResponse(response: Response): never {
|
|
2272
|
+
if (response.status === 401 || response.status === 403) {
|
|
2273
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2274
|
+
}
|
|
2275
|
+
throw new SyncFailure(
|
|
2276
|
+
response.status >= 500 || response.status === 429
|
|
2277
|
+
? "provider_unavailable"
|
|
2278
|
+
: "provider_rejected",
|
|
2279
|
+
response.status >= 500 || response.status === 429,
|
|
2280
|
+
);
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
function selectedDriveSource(
|
|
2284
|
+
metadata: ReturnType<typeof GoogleDriveConnectionMetadata.parse>,
|
|
2285
|
+
externalSourceId: string,
|
|
2286
|
+
): GoogleDriveSelectedSource | null {
|
|
2287
|
+
const sources =
|
|
2288
|
+
metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
|
|
2289
|
+
return sources.find((source) => source.id === externalSourceId) ?? null;
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
export function googleDriveSyncProviderAccessAllowed(input: {
|
|
2293
|
+
connectionVersion: number;
|
|
2294
|
+
resolvedConnectionVersion: number | undefined;
|
|
2295
|
+
connectionStatus: string;
|
|
2296
|
+
lifecycleState: string | undefined;
|
|
2297
|
+
grantedScopes: string[];
|
|
2298
|
+
}): boolean {
|
|
2299
|
+
return (
|
|
2300
|
+
input.resolvedConnectionVersion !== undefined &&
|
|
2301
|
+
input.connectionVersion === input.resolvedConnectionVersion &&
|
|
2302
|
+
input.connectionStatus === "active" &&
|
|
2303
|
+
(input.lifecycleState === undefined || input.lifecycleState === "active") &&
|
|
2304
|
+
googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync")
|
|
2305
|
+
);
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
async function listDriveChildren(
|
|
2309
|
+
input: { folderId: string; driveId: string | null; pageToken: string | null; pageSize: number },
|
|
2310
|
+
authorization: string | undefined,
|
|
2311
|
+
): Promise<GoogleDriveInventoryPage> {
|
|
2312
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2313
|
+
const url = new URL(`${DRIVE_API_BASE}/files`);
|
|
2314
|
+
url.searchParams.set(
|
|
2315
|
+
"q",
|
|
2316
|
+
`'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`,
|
|
2317
|
+
);
|
|
2318
|
+
url.searchParams.set("pageSize", String(input.pageSize));
|
|
2319
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2320
|
+
url.searchParams.set("includeItemsFromAllDrives", "true");
|
|
2321
|
+
url.searchParams.set(
|
|
2322
|
+
"fields",
|
|
2323
|
+
"nextPageToken,incompleteSearch,files(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed)",
|
|
2324
|
+
);
|
|
2325
|
+
if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
|
|
2326
|
+
if (input.driveId) {
|
|
2327
|
+
url.searchParams.set("corpora", "drive");
|
|
2328
|
+
url.searchParams.set("driveId", input.driveId);
|
|
2329
|
+
}
|
|
2330
|
+
const response = await fetch(url, {
|
|
2331
|
+
headers: { authorization, accept: "application/json" },
|
|
2332
|
+
redirect: "error",
|
|
2333
|
+
signal: AbortSignal.timeout(15_000),
|
|
2334
|
+
});
|
|
2335
|
+
if (!response.ok) {
|
|
2336
|
+
if (response.status === 401 || response.status === 403) {
|
|
2337
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2338
|
+
}
|
|
2339
|
+
throw new SyncFailure(
|
|
2340
|
+
"provider_unavailable",
|
|
2341
|
+
response.status >= 500 || response.status === 429,
|
|
2342
|
+
);
|
|
2343
|
+
}
|
|
2344
|
+
const payload = (await readResponseJsonBounded(
|
|
2345
|
+
response,
|
|
2346
|
+
DRIVE_JSON_MAX_BYTES,
|
|
2347
|
+
"Google Drive file list",
|
|
2348
|
+
)) as Record<string, unknown>;
|
|
2349
|
+
const rawFiles = Array.isArray(payload.files) ? payload.files : [];
|
|
2350
|
+
return {
|
|
2351
|
+
items: rawFiles.map(parseDriveProviderItem),
|
|
2352
|
+
nextPageToken: typeof payload.nextPageToken === "string" ? payload.nextPageToken : null,
|
|
2353
|
+
incompleteSearch: payload.incompleteSearch === true,
|
|
2354
|
+
};
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
function parseDriveProviderItem(value: unknown): GoogleDriveInventoryProviderItem {
|
|
2358
|
+
const row = value && typeof value === "object" ? (value as Record<string, unknown>) : {};
|
|
2359
|
+
return {
|
|
2360
|
+
id: requiredString(row.id, "id"),
|
|
2361
|
+
name: requiredString(row.name, "name"),
|
|
2362
|
+
mimeType: requiredString(row.mimeType, "mimeType"),
|
|
2363
|
+
driveId: optionalString(row.driveId),
|
|
2364
|
+
parents: Array.isArray(row.parents)
|
|
2365
|
+
? row.parents.filter((item): item is string => typeof item === "string")
|
|
2366
|
+
: [],
|
|
2367
|
+
modifiedTime: optionalString(row.modifiedTime),
|
|
2368
|
+
createdTime: optionalString(row.createdTime),
|
|
2369
|
+
version: optionalString(row.version),
|
|
2370
|
+
md5Checksum: optionalString(row.md5Checksum),
|
|
2371
|
+
size: optionalString(row.size),
|
|
2372
|
+
webViewLink: optionalString(row.webViewLink),
|
|
2373
|
+
trashed: row.trashed === true,
|
|
2374
|
+
};
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
async function fetchDriveBytes(
|
|
2378
|
+
entry: GoogleDriveInventoryEntry,
|
|
2379
|
+
authorization: string | undefined,
|
|
2380
|
+
maxBytes: number,
|
|
2381
|
+
): Promise<Uint8Array> {
|
|
2382
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2383
|
+
const url =
|
|
2384
|
+
entry.transfer.action === "export"
|
|
2385
|
+
? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`)
|
|
2386
|
+
: new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
|
|
2387
|
+
if (entry.transfer.action === "export") {
|
|
2388
|
+
url.searchParams.set("mimeType", entry.transfer.contentType);
|
|
2389
|
+
} else {
|
|
2390
|
+
url.searchParams.set("alt", "media");
|
|
2391
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2392
|
+
}
|
|
2393
|
+
const response = await fetch(url, {
|
|
2394
|
+
headers: { authorization },
|
|
2395
|
+
redirect: "error",
|
|
2396
|
+
signal: AbortSignal.timeout(30_000),
|
|
2397
|
+
});
|
|
2398
|
+
if (!response.ok) {
|
|
2399
|
+
if (response.status === 401 || response.status === 403) {
|
|
2400
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2401
|
+
}
|
|
2402
|
+
throw new SyncFailure(
|
|
2403
|
+
"provider_unavailable",
|
|
2404
|
+
response.status >= 500 || response.status === 429,
|
|
2405
|
+
);
|
|
2406
|
+
}
|
|
2407
|
+
const declared = Number(response.headers.get("content-length") ?? "0");
|
|
2408
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
2409
|
+
throw new SyncFailure("content_too_large", false);
|
|
2410
|
+
}
|
|
2411
|
+
if (!response.body) {
|
|
2412
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
2413
|
+
if (bytes.byteLength > maxBytes) throw new SyncFailure("content_too_large", false);
|
|
2414
|
+
return bytes;
|
|
2415
|
+
}
|
|
2416
|
+
const reader = response.body.getReader();
|
|
2417
|
+
const chunks: Uint8Array[] = [];
|
|
2418
|
+
let total = 0;
|
|
2419
|
+
while (true) {
|
|
2420
|
+
const { done, value } = await reader.read();
|
|
2421
|
+
if (done) break;
|
|
2422
|
+
total += value.byteLength;
|
|
2423
|
+
if (total > maxBytes) {
|
|
2424
|
+
await reader.cancel();
|
|
2425
|
+
throw new SyncFailure("content_too_large", false);
|
|
2426
|
+
}
|
|
2427
|
+
chunks.push(value);
|
|
2428
|
+
}
|
|
2429
|
+
const bytes = new Uint8Array(total);
|
|
2430
|
+
let offset = 0;
|
|
2431
|
+
for (const chunk of chunks) {
|
|
2432
|
+
bytes.set(chunk, offset);
|
|
2433
|
+
offset += chunk.byteLength;
|
|
2434
|
+
}
|
|
2435
|
+
return bytes;
|
|
2436
|
+
}
|
|
2437
|
+
|
|
2438
|
+
async function listAtlassianItems(
|
|
2439
|
+
source: AtlassianSelectedSource,
|
|
2440
|
+
cursor: string | null,
|
|
2441
|
+
pageSize: number,
|
|
2442
|
+
authorization: string | undefined,
|
|
2443
|
+
): Promise<AtlassianInventoryPage> {
|
|
2444
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2445
|
+
if (source.kind === "jira_project") {
|
|
2446
|
+
const url = new URL(
|
|
2447
|
+
`${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/search/jql`,
|
|
2448
|
+
);
|
|
2449
|
+
url.searchParams.set(
|
|
2450
|
+
"jql",
|
|
2451
|
+
`project = "${source.key.replaceAll('"', '\\"')}" ORDER BY key ASC`,
|
|
2452
|
+
);
|
|
2453
|
+
url.searchParams.set("maxResults", String(pageSize));
|
|
2454
|
+
url.searchParams.set("fields", "summary,created,updated");
|
|
2455
|
+
if (cursor) url.searchParams.set("nextPageToken", cursor);
|
|
2456
|
+
const payload = objectRecord(await atlassianJsonRequest(url, authorization, "Jira issue list"));
|
|
2457
|
+
const issues = Array.isArray(payload.issues) ? payload.issues : [];
|
|
2458
|
+
return {
|
|
2459
|
+
items: issues.map((raw) => {
|
|
2460
|
+
const row = objectRecord(raw);
|
|
2461
|
+
const fields = objectRecord(row.fields);
|
|
2462
|
+
const id = requiredAtlassianString(row.id, "issue.id");
|
|
2463
|
+
const key = requiredAtlassianString(row.key, "issue.key");
|
|
2464
|
+
const updatedAt = optionalString(fields.updated);
|
|
2465
|
+
return {
|
|
2466
|
+
id,
|
|
2467
|
+
key,
|
|
2468
|
+
title: requiredAtlassianString(fields.summary, "issue.summary"),
|
|
2469
|
+
version: updatedAt,
|
|
2470
|
+
createdAt: optionalString(fields.created),
|
|
2471
|
+
updatedAt,
|
|
2472
|
+
webUrl: new URL(`/browse/${encodeURIComponent(key)}`, source.siteUrl).toString(),
|
|
2473
|
+
};
|
|
2474
|
+
}),
|
|
2475
|
+
nextCursor: optionalString(payload.nextPageToken),
|
|
2476
|
+
};
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
const url = new URL(
|
|
2480
|
+
`${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/spaces/${encodeURIComponent(source.resourceId)}/pages`,
|
|
2481
|
+
);
|
|
2482
|
+
url.searchParams.set("limit", String(pageSize));
|
|
2483
|
+
url.searchParams.set("sort", "id");
|
|
2484
|
+
if (cursor) url.searchParams.set("cursor", cursor);
|
|
2485
|
+
const payload = objectRecord(
|
|
2486
|
+
await atlassianJsonRequest(url, authorization, "Confluence page list"),
|
|
2487
|
+
);
|
|
2488
|
+
const pages = Array.isArray(payload.results) ? payload.results : [];
|
|
2489
|
+
return {
|
|
2490
|
+
items: pages.map((raw) => {
|
|
2491
|
+
const row = objectRecord(raw);
|
|
2492
|
+
const version = objectRecord(row.version);
|
|
2493
|
+
const id = requiredAtlassianString(row.id, "page.id");
|
|
2494
|
+
const webUi = optionalString(objectRecord(row._links).webui);
|
|
2495
|
+
const versionNumber = numberValue(version.number);
|
|
2496
|
+
return {
|
|
2497
|
+
id,
|
|
2498
|
+
key: id,
|
|
2499
|
+
title: requiredAtlassianString(row.title, "page.title"),
|
|
2500
|
+
version: versionNumber === undefined ? null : String(versionNumber),
|
|
2501
|
+
createdAt: optionalString(row.createdAt),
|
|
2502
|
+
updatedAt: optionalString(version.createdAt),
|
|
2503
|
+
webUrl: webUi
|
|
2504
|
+
? new URL(webUi.startsWith("/wiki/") ? webUi : `/wiki${webUi}`, source.siteUrl).toString()
|
|
2505
|
+
: new URL(
|
|
2506
|
+
`/wiki/spaces/${encodeURIComponent(source.key)}/pages/${encodeURIComponent(id)}`,
|
|
2507
|
+
source.siteUrl,
|
|
2508
|
+
).toString(),
|
|
2509
|
+
};
|
|
2510
|
+
}),
|
|
2511
|
+
nextCursor: confluenceCursor(objectRecord(payload._links).next),
|
|
2512
|
+
};
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
async function fetchAtlassianContent(
|
|
2516
|
+
source: AtlassianSelectedSource,
|
|
2517
|
+
entry: AtlassianInventoryEntry,
|
|
2518
|
+
authorization: string | undefined,
|
|
2519
|
+
maxBytes: number,
|
|
2520
|
+
): Promise<Uint8Array> {
|
|
2521
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2522
|
+
const objectId = entry.externalObjectId.slice(entry.externalObjectId.indexOf(":") + 1);
|
|
2523
|
+
const markdown =
|
|
2524
|
+
source.kind === "jira_project"
|
|
2525
|
+
? await jiraIssueMarkdown(source, objectId, entry, authorization)
|
|
2526
|
+
: await confluencePageMarkdown(source, objectId, entry, authorization);
|
|
2527
|
+
const bytes = new TextEncoder().encode(markdown);
|
|
2528
|
+
if (bytes.byteLength > maxBytes) throw new SyncFailure("content_too_large", false);
|
|
2529
|
+
return bytes;
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
async function jiraIssueMarkdown(
|
|
2533
|
+
source: AtlassianSelectedSource,
|
|
2534
|
+
issueId: string,
|
|
2535
|
+
entry: AtlassianInventoryEntry,
|
|
2536
|
+
authorization: string,
|
|
2537
|
+
): Promise<string> {
|
|
2538
|
+
const issueUrl = new URL(
|
|
2539
|
+
`${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/issue/${encodeURIComponent(issueId)}`,
|
|
2540
|
+
);
|
|
2541
|
+
issueUrl.searchParams.set(
|
|
2542
|
+
"fields",
|
|
2543
|
+
"summary,description,status,issuetype,priority,assignee,reporter,labels,created,updated",
|
|
2544
|
+
);
|
|
2545
|
+
const issue = objectRecord(await atlassianJsonRequest(issueUrl, authorization, "Jira issue"));
|
|
2546
|
+
const fields = objectRecord(issue.fields);
|
|
2547
|
+
const lines = [
|
|
2548
|
+
`# ${entry.title}`,
|
|
2549
|
+
"",
|
|
2550
|
+
`- Key: ${optionalString(issue.key) ?? entry.title}`,
|
|
2551
|
+
`- Type: ${optionalString(objectRecord(fields.issuetype).name) ?? "Unknown"}`,
|
|
2552
|
+
`- Status: ${optionalString(objectRecord(fields.status).name) ?? "Unknown"}`,
|
|
2553
|
+
`- Priority: ${optionalString(objectRecord(fields.priority).name) ?? "None"}`,
|
|
2554
|
+
`- Assignee: ${optionalString(objectRecord(fields.assignee).displayName) ?? "Unassigned"}`,
|
|
2555
|
+
`- Reporter: ${optionalString(objectRecord(fields.reporter).displayName) ?? "Unknown"}`,
|
|
2556
|
+
`- Updated: ${optionalString(fields.updated) ?? "Unknown"}`,
|
|
2557
|
+
`- Source: ${entry.sourceUri}`,
|
|
2558
|
+
"",
|
|
2559
|
+
"## Description",
|
|
2560
|
+
"",
|
|
2561
|
+
adfToMarkdown(fields.description) || "No description.",
|
|
2562
|
+
];
|
|
2563
|
+
const labels = Array.isArray(fields.labels)
|
|
2564
|
+
? fields.labels.filter((value): value is string => typeof value === "string")
|
|
2565
|
+
: [];
|
|
2566
|
+
if (labels.length > 0) lines.splice(9, 0, `- Labels: ${labels.join(", ")}`);
|
|
2567
|
+
const comments = await jiraComments(source, issueId, authorization);
|
|
2568
|
+
if (comments.length > 0) {
|
|
2569
|
+
lines.push("", "## Comments", "");
|
|
2570
|
+
for (const comment of comments) {
|
|
2571
|
+
const row = objectRecord(comment);
|
|
2572
|
+
const author = optionalString(objectRecord(row.author).displayName) ?? "Unknown author";
|
|
2573
|
+
const created = optionalString(row.created) ?? "Unknown date";
|
|
2574
|
+
lines.push(`### ${author} · ${created}`, "", adfToMarkdown(row.body), "");
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
return lines.join("\n");
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
async function jiraComments(
|
|
2581
|
+
source: AtlassianSelectedSource,
|
|
2582
|
+
issueId: string,
|
|
2583
|
+
authorization: string,
|
|
2584
|
+
): Promise<unknown[]> {
|
|
2585
|
+
const comments: unknown[] = [];
|
|
2586
|
+
let startAt = 0;
|
|
2587
|
+
for (let page = 0; page < 20; page += 1) {
|
|
2588
|
+
const url = new URL(
|
|
2589
|
+
`${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/issue/${encodeURIComponent(issueId)}/comment`,
|
|
2590
|
+
);
|
|
2591
|
+
url.searchParams.set("startAt", String(startAt));
|
|
2592
|
+
url.searchParams.set("maxResults", "100");
|
|
2593
|
+
url.searchParams.set("orderBy", "created");
|
|
2594
|
+
const payload = objectRecord(await atlassianJsonRequest(url, authorization, "Jira comments"));
|
|
2595
|
+
const pageComments = Array.isArray(payload.comments) ? payload.comments : [];
|
|
2596
|
+
comments.push(...pageComments);
|
|
2597
|
+
const total = numberValue(payload.total) ?? comments.length;
|
|
2598
|
+
if (comments.length >= total || pageComments.length === 0) break;
|
|
2599
|
+
startAt += pageComments.length;
|
|
2600
|
+
}
|
|
2601
|
+
return comments;
|
|
2602
|
+
}
|
|
2603
|
+
|
|
2604
|
+
async function confluencePageMarkdown(
|
|
2605
|
+
source: AtlassianSelectedSource,
|
|
2606
|
+
pageId: string,
|
|
2607
|
+
entry: AtlassianInventoryEntry,
|
|
2608
|
+
authorization: string,
|
|
2609
|
+
): Promise<string> {
|
|
2610
|
+
const pageUrl = new URL(
|
|
2611
|
+
`${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/pages/${encodeURIComponent(pageId)}`,
|
|
2612
|
+
);
|
|
2613
|
+
pageUrl.searchParams.set("body-format", "storage");
|
|
2614
|
+
const page = objectRecord(await atlassianJsonRequest(pageUrl, authorization, "Confluence page"));
|
|
2615
|
+
const storage = objectRecord(objectRecord(page.body).storage);
|
|
2616
|
+
const lines = [
|
|
2617
|
+
`# ${entry.title}`,
|
|
2618
|
+
"",
|
|
2619
|
+
`- Space: ${source.name} (${source.key})`,
|
|
2620
|
+
`- Updated: ${entry.modifiedTime ?? "Unknown"}`,
|
|
2621
|
+
`- Source: ${entry.sourceUri}`,
|
|
2622
|
+
"",
|
|
2623
|
+
"## Content",
|
|
2624
|
+
"",
|
|
2625
|
+
optionalString(storage.value) ?? "No content.",
|
|
2626
|
+
];
|
|
2627
|
+
const comments = await confluenceFooterComments(source, pageId, authorization);
|
|
2628
|
+
if (comments.length > 0) {
|
|
2629
|
+
lines.push("", "## Comments", "");
|
|
2630
|
+
for (const comment of comments) {
|
|
2631
|
+
const row = objectRecord(comment);
|
|
2632
|
+
const body = objectRecord(objectRecord(row.body).storage);
|
|
2633
|
+
const version = objectRecord(row.version);
|
|
2634
|
+
lines.push(
|
|
2635
|
+
`### Comment · ${optionalString(version.createdAt) ?? "Unknown date"}`,
|
|
2636
|
+
"",
|
|
2637
|
+
optionalString(body.value) ?? "",
|
|
2638
|
+
"",
|
|
2639
|
+
);
|
|
2640
|
+
}
|
|
2641
|
+
}
|
|
2642
|
+
return lines.join("\n");
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
async function confluenceFooterComments(
|
|
2646
|
+
source: AtlassianSelectedSource,
|
|
2647
|
+
pageId: string,
|
|
2648
|
+
authorization: string,
|
|
2649
|
+
): Promise<unknown[]> {
|
|
2650
|
+
const comments: unknown[] = [];
|
|
2651
|
+
let cursor: string | null = null;
|
|
2652
|
+
for (let page = 0; page < 20; page += 1) {
|
|
2653
|
+
const url = new URL(
|
|
2654
|
+
`${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/pages/${encodeURIComponent(pageId)}/footer-comments`,
|
|
2655
|
+
);
|
|
2656
|
+
url.searchParams.set("body-format", "storage");
|
|
2657
|
+
url.searchParams.set("limit", "100");
|
|
2658
|
+
if (cursor) url.searchParams.set("cursor", cursor);
|
|
2659
|
+
const payload = objectRecord(
|
|
2660
|
+
await atlassianJsonRequest(url, authorization, "Confluence page comments"),
|
|
2661
|
+
);
|
|
2662
|
+
const results = Array.isArray(payload.results) ? payload.results : [];
|
|
2663
|
+
comments.push(...results);
|
|
2664
|
+
cursor = confluenceCursor(objectRecord(payload._links).next);
|
|
2665
|
+
if (!cursor) break;
|
|
2666
|
+
}
|
|
2667
|
+
return comments;
|
|
2668
|
+
}
|
|
2669
|
+
|
|
2670
|
+
async function atlassianJsonRequest(
|
|
2671
|
+
url: URL,
|
|
2672
|
+
authorization: string,
|
|
2673
|
+
label: string,
|
|
2674
|
+
): Promise<unknown> {
|
|
2675
|
+
const response = await fetch(url, {
|
|
2676
|
+
headers: { authorization, accept: "application/json" },
|
|
2677
|
+
redirect: "error",
|
|
2678
|
+
signal: AbortSignal.timeout(20_000),
|
|
2679
|
+
});
|
|
2680
|
+
if (!response.ok) {
|
|
2681
|
+
await response.body?.cancel().catch(() => undefined);
|
|
2682
|
+
if (response.status === 401 || response.status === 403) {
|
|
2683
|
+
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2684
|
+
}
|
|
2685
|
+
throw new SyncFailure(
|
|
2686
|
+
"provider_unavailable",
|
|
2687
|
+
response.status >= 500 || response.status === 429,
|
|
2688
|
+
);
|
|
2689
|
+
}
|
|
2690
|
+
return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES, label);
|
|
2691
|
+
}
|
|
2692
|
+
|
|
2693
|
+
export function confluenceCursor(value: unknown): string | null {
|
|
2694
|
+
const next = optionalString(value);
|
|
2695
|
+
if (!next) return null;
|
|
2696
|
+
const url = new URL(next, ATLASSIAN_API_BASE);
|
|
2697
|
+
const cursor = url.searchParams.get("cursor");
|
|
2698
|
+
if (!cursor || cursor.length > 4_096 || /[\u0000-\u001f]/.test(cursor)) {
|
|
2699
|
+
throw new Error("invalid Confluence cursor");
|
|
2700
|
+
}
|
|
2701
|
+
return cursor;
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
export function adfToMarkdown(value: unknown): string {
|
|
2705
|
+
const node = objectRecord(value);
|
|
2706
|
+
const type = optionalString(node.type);
|
|
2707
|
+
const children = Array.isArray(node.content) ? node.content.map(adfToMarkdown).join("") : "";
|
|
2708
|
+
if (type === "text") {
|
|
2709
|
+
let text = typeof node.text === "string" ? node.text : "";
|
|
2710
|
+
const marks = Array.isArray(node.marks) ? node.marks : [];
|
|
2711
|
+
for (const rawMark of marks) {
|
|
2712
|
+
const mark = objectRecord(rawMark);
|
|
2713
|
+
const markType = optionalString(mark.type);
|
|
2714
|
+
if (markType === "code") text = `\`${text}\``;
|
|
2715
|
+
if (markType === "strong") text = `**${text}**`;
|
|
2716
|
+
if (markType === "em") text = `*${text}*`;
|
|
2717
|
+
if (markType === "strike") text = `~~${text}~~`;
|
|
2718
|
+
if (markType === "link") {
|
|
2719
|
+
const href = optionalString(objectRecord(mark.attrs).href);
|
|
2720
|
+
if (href) text = `[${text}](${href})`;
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
return text;
|
|
2724
|
+
}
|
|
2725
|
+
if (type === "paragraph") return `${children}\n\n`;
|
|
2726
|
+
if (type === "heading") {
|
|
2727
|
+
const level = Math.min(Math.max(numberValue(objectRecord(node.attrs).level) ?? 2, 1), 6);
|
|
2728
|
+
return `${"#".repeat(level)} ${children.trim()}\n\n`;
|
|
2729
|
+
}
|
|
2730
|
+
if (type === "hardBreak") return " \n";
|
|
2731
|
+
if (type === "rule") return "---\n\n";
|
|
2732
|
+
if (type === "blockquote") {
|
|
2733
|
+
return `${children
|
|
2734
|
+
.trim()
|
|
2735
|
+
.split("\n")
|
|
2736
|
+
.map((line) => `> ${line}`)
|
|
2737
|
+
.join("\n")}\n\n`;
|
|
2738
|
+
}
|
|
2739
|
+
if (type === "bulletList") {
|
|
2740
|
+
return `${children.trimEnd()}\n`;
|
|
2741
|
+
}
|
|
2742
|
+
if (type === "orderedList") {
|
|
2743
|
+
return `${children.trimEnd()}\n`;
|
|
2744
|
+
}
|
|
2745
|
+
if (type === "listItem") {
|
|
2746
|
+
return `- ${children.trim().replaceAll("\n", "\n ")}\n`;
|
|
2747
|
+
}
|
|
2748
|
+
if (type === "codeBlock") return `\`\`\`\n${children.trimEnd()}\n\`\`\`\n\n`;
|
|
2749
|
+
if (type === "mention") return `@${optionalString(objectRecord(node.attrs).text) ?? "user"}`;
|
|
2750
|
+
if (type === "emoji") return optionalString(objectRecord(node.attrs).text) ?? "";
|
|
2751
|
+
if (type === "inlineCard") {
|
|
2752
|
+
const url = optionalString(objectRecord(node.attrs).url);
|
|
2753
|
+
return url ? `[${url}](${url})` : "";
|
|
2754
|
+
}
|
|
2755
|
+
return children;
|
|
2756
|
+
}
|
|
2757
|
+
|
|
2758
|
+
function objectRecord(value: unknown): Record<string, unknown> {
|
|
2759
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
2760
|
+
? (value as Record<string, unknown>)
|
|
2761
|
+
: {};
|
|
2762
|
+
}
|
|
2763
|
+
|
|
2764
|
+
function requiredAtlassianString(value: unknown, label: string): string {
|
|
2765
|
+
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${label}`);
|
|
2766
|
+
return value;
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
function deterministicUuid(value: string): string {
|
|
2770
|
+
const bytes = createHash("sha256").update(value).digest().subarray(0, 16);
|
|
2771
|
+
bytes[6] = (bytes[6]! & 0x0f) | 0x50;
|
|
2772
|
+
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
|
2773
|
+
const hex = bytes.toString("hex");
|
|
2774
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
2775
|
+
}
|
|
2776
|
+
|
|
2777
|
+
function safeFilename(value: string): string {
|
|
2778
|
+
const normalized = value
|
|
2779
|
+
.normalize("NFKC")
|
|
2780
|
+
.replace(/[\\/:*?"<>|\0-\x1f]/g, "_")
|
|
2781
|
+
.trim();
|
|
2782
|
+
return (normalized || "document").slice(0, 240);
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
function requiredString(value: unknown, label: string): string {
|
|
2786
|
+
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${label}`);
|
|
2787
|
+
return value;
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2790
|
+
function optionalString(value: unknown): string | null {
|
|
2791
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
function numberValue(value: unknown): number | undefined {
|
|
2795
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
function boundedError(error: unknown): string {
|
|
2799
|
+
return (error instanceof Error ? error.message : String(error)).slice(0, 1000);
|
|
2800
|
+
}
|
|
2801
|
+
|
|
2802
|
+
function entryMetadataHash(entry: KnowledgeSyncEntry): string {
|
|
2803
|
+
return createHash("sha256")
|
|
2804
|
+
.update(
|
|
2805
|
+
JSON.stringify({
|
|
2806
|
+
externalObjectId: entry.externalObjectId,
|
|
2807
|
+
externalVersionId: entry.externalVersionId,
|
|
2808
|
+
title: entry.title,
|
|
2809
|
+
mimeType: entry.mimeType,
|
|
2810
|
+
driveId: entry.driveId,
|
|
2811
|
+
modifiedTime: entry.modifiedTime,
|
|
2812
|
+
createdTime: entry.createdTime,
|
|
2813
|
+
sourceUri: entry.sourceUri,
|
|
2814
|
+
}),
|
|
2815
|
+
)
|
|
2816
|
+
.digest("hex");
|
|
2817
|
+
}
|
|
2818
|
+
|
|
2819
|
+
function inventoryLimit(
|
|
2820
|
+
reason: GoogleDriveInventoryStopReason | null,
|
|
2821
|
+
): KnowledgeSourceSyncRunSummary["limitReached"] {
|
|
2822
|
+
if (reason === "api_request_limit") return "provider_requests";
|
|
2823
|
+
if (reason === "elapsed_time_limit") return "elapsed_time";
|
|
2824
|
+
if (reason === "item_limit") return "items";
|
|
2825
|
+
if (reason === "known_byte_limit") return "bytes";
|
|
2826
|
+
return null;
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
type SyncFailureCode = KnowledgeSourceSyncRunSummary["failures"][number]["code"];
|
|
2830
|
+
|
|
2831
|
+
class SyncFailure extends Error {
|
|
2832
|
+
constructor(
|
|
2833
|
+
readonly code: SyncFailureCode,
|
|
2834
|
+
readonly retryable: boolean,
|
|
2835
|
+
readonly reconnectRequired = false,
|
|
2836
|
+
) {
|
|
2837
|
+
super(code);
|
|
2838
|
+
this.name = "KnowledgeSourceSyncFailure";
|
|
2839
|
+
}
|
|
2840
|
+
}
|