@mycadet/assistant 0.12.10 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
- package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
- package/node_modules/@mycadet/ces-client/src/http-credentials.ts +18 -1
- package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
- package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
- package/node_modules/@mycadet/gateway-client/src/gateway-ipc-contracts.ts +0 -20
- package/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
- package/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
- package/openapi.yaml +2 -708
- package/package.json +2 -3
- package/scripts/sync-web-search-catalog.ts +1 -2
- package/src/__tests__/cloud-profile-guard.test.ts +41 -5
- package/src/__tests__/config-schema.test.ts +6 -38
- package/src/__tests__/conversation-queue.test.ts +45 -6
- package/src/__tests__/db-readiness-http-gate.test.ts +0 -6
- package/src/__tests__/persist-onboarding-artifacts.test.ts +0 -1
- package/src/__tests__/persona-resolver.test.ts +0 -51
- package/src/__tests__/provider-cloud-proxy-integration.test.ts +2 -110
- package/src/__tests__/run-due-schedules.test.ts +0 -4
- package/src/__tests__/schedule-store.test.ts +0 -12
- package/src/__tests__/stt-stream-session.test.ts +4 -4
- package/src/__tests__/telephony-tts-guard.test.ts +2 -50
- package/src/__tests__/voice-config-update.test.ts +12 -167
- package/src/__tests__/web-search-catalog-parity.test.ts +0 -2
- package/src/__tests__/workspace-migration-130-speech-mode-to-provider.test.ts +9 -7
- package/src/__tests__/workspace-migration-132-web-search-mode-to-provider.test.ts +6 -6
- package/src/calls/__tests__/voice-session-bridge.test.ts +1 -4
- package/src/calls/resolve-call-tts-provider.ts +6 -25
- package/src/calls/telephony-credential-preflight.ts +0 -10
- package/src/calls/telephony-tts-capability.ts +1 -17
- package/src/cli/commands/tts.ts +1 -2
- package/src/cli/index.help.ts +0 -2
- package/src/cli/program.ts +0 -2
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +4 -42
- package/src/config/schemas/__tests__/stt.test.ts +10 -14
- package/src/config/schemas/llm.ts +3 -5
- package/src/config/schemas/services.ts +5 -6
- package/src/config/schemas/stt.ts +1 -3
- package/src/config/schemas/tts.ts +1 -30
- package/src/daemon/daemon-readiness.ts +4 -29
- package/src/daemon/lifecycle.ts +0 -31
- package/src/daemon/providers-setup.ts +0 -8
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
- package/src/heartbeat/heartbeat-service.ts +1 -66
- package/src/ipc/__tests__/db-readiness-ipc-gate.test.ts +1 -9
- package/src/live-voice/__tests__/live-voice-credential-preflight.test.ts +0 -44
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +4 -6
- package/src/live-voice/__tests__/live-voice-stt.test.ts +1 -1
- package/src/live-voice/live-voice-credential-preflight.ts +9 -33
- package/src/live-voice/live-voice-session.ts +10 -6
- package/src/live-voice/live-voice-tts.ts +1 -8
- package/src/plugins/defaults/memory/worker.ts +3 -3
- package/src/prompts/persona-resolver.ts +0 -35
- package/src/providers/__tests__/search-provider-catalog.test.ts +5 -8
- package/src/providers/__tests__/voice-error-copy.test.ts +9 -10
- package/src/providers/search-provider-catalog.ts +7 -39
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +3 -47
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +6 -372
- package/src/providers/speech-to-text/provider-catalog.ts +2 -73
- package/src/providers/speech-to-text/resolve.ts +3 -120
- package/src/runtime/http-router-types.ts +1 -3
- package/src/runtime/http-server.ts +1 -1
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +13 -11
- package/src/runtime/routes/__tests__/tts-routes.test.ts +2 -6
- package/src/runtime/routes/conversation-query-routes.ts +0 -96
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +7 -7
- package/src/runtime/routes/index.ts +0 -4
- package/src/runtime/routes/live-voice-routes.test.ts +4 -11
- package/src/runtime/routes/live-voice-routes.ts +8 -11
- package/src/runtime/routes/secret-routes.ts +0 -6
- package/src/runtime/routes/tts-routes.ts +0 -38
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1 -7
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +0 -4
- package/src/schedule/schedule-store.ts +0 -7
- package/src/schedule/scheduler.ts +0 -5
- package/src/schedule/worker.ts +1 -1
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +2 -10
- package/src/stt/daemon-batch-transcriber.ts +4 -38
- package/src/stt/types.ts +1 -9
- package/src/tools/network/__tests__/web-search.test.ts +3 -250
- package/src/tools/network/web-search.ts +4 -165
- package/src/tts/__tests__/provider-resolution.test.ts +0 -5
- package/src/tts/provider-catalog.ts +0 -4
- package/src/tts/types.ts +0 -1
- package/src/__tests__/cloud-speech-defaults.test.ts +0 -330
- package/src/__tests__/migration-cross-version-compatibility.test.ts +0 -1708
- package/src/__tests__/migration-export-http.test.ts +0 -557
- package/src/__tests__/migration-export-streaming.test.ts +0 -350
- package/src/__tests__/migration-export-to-gcs.test.ts +0 -706
- package/src/__tests__/migration-import-commit-http.test.ts +0 -1176
- package/src/__tests__/migration-import-from-gcs.test.ts +0 -534
- package/src/__tests__/migration-import-from-path.test.ts +0 -324
- package/src/__tests__/migration-import-from-url.test.ts +0 -588
- package/src/__tests__/migration-import-preflight-http.test.ts +0 -739
- package/src/__tests__/migration-jobs-status.test.ts +0 -139
- package/src/__tests__/migration-parity-persistence.test.ts +0 -1891
- package/src/__tests__/migration-transport.test.ts +0 -1089
- package/src/__tests__/migration-validate-http.test.ts +0 -634
- package/src/__tests__/migration-wizard.test.ts +0 -1395
- package/src/__tests__/rebind-secrets-screen.test.ts +0 -876
- package/src/__tests__/sanitize-config-for-transfer.test.ts +0 -259
- package/src/__tests__/transfer-progress-screen.test.ts +0 -1217
- package/src/__tests__/validation-results-screen.test.ts +0 -1130
- package/src/__tests__/vbundle-pax-and-symlink.test.ts +0 -147
- package/src/background-wake/background-wake-routes.test.ts +0 -906
- package/src/background-wake/cloud-client.test.ts +0 -328
- package/src/background-wake/cloud-client.ts +0 -196
- package/src/background-wake/next-wake.test.ts +0 -335
- package/src/background-wake/next-wake.ts +0 -188
- package/src/background-wake/publisher.ts +0 -93
- package/src/background-wake/wake-intent-hooks.test.ts +0 -234
- package/src/cli/commands/__tests__/backup.test.ts +0 -363
- package/src/cli/commands/backup.help.ts +0 -180
- package/src/cli/commands/backup.ts +0 -398
- package/src/cloud/cloud-speech.test.ts +0 -335
- package/src/cloud/cloud-speech.ts +0 -300
- package/src/config/cloud-speech-defaults.ts +0 -300
- package/src/config/sanitize-for-transfer.ts +0 -111
- package/src/live-voice/__tests__/live-voice-cloud-speech-fallback.test.ts +0 -250
- package/src/providers/speech-to-text/__tests__/cadet-cloud-realtime.test.ts +0 -457
- package/src/providers/speech-to-text/__tests__/cadet-cloud.test.ts +0 -271
- package/src/providers/speech-to-text/__tests__/cadet-speech-relay-connection.test.ts +0 -126
- package/src/providers/speech-to-text/cadet-cloud-flux-realtime.ts +0 -309
- package/src/providers/speech-to-text/cadet-cloud-realtime.ts +0 -315
- package/src/providers/speech-to-text/cadet-cloud.ts +0 -108
- package/src/providers/speech-to-text/cadet-speech-relay-connection.ts +0 -176
- package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +0 -254
- package/src/runtime/migrations/__tests__/job-registry.test.ts +0 -346
- package/src/runtime/migrations/__tests__/origin-mode.test.ts +0 -62
- package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +0 -184
- package/src/runtime/migrations/__tests__/staged-import-path.test.ts +0 -110
- package/src/runtime/migrations/__tests__/v1-test-helpers.ts +0 -112
- package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +0 -285
- package/src/runtime/migrations/__tests__/vbundle-builder-fd-leak.test.ts +0 -126
- package/src/runtime/migrations/__tests__/vbundle-builder-v1-shape.test.ts +0 -253
- package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +0 -208
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +0 -485
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +0 -349
- package/src/runtime/migrations/__tests__/vbundle-import-version-compat.test.ts +0 -196
- package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +0 -551
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +0 -428
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +0 -220
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +0 -1823
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +0 -598
- package/src/runtime/migrations/__tests__/vbundle-symlink-importer.test.ts +0 -463
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -538
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming.test.ts +0 -515
- package/src/runtime/migrations/__tests__/vbundle-symlink-tar.test.ts +0 -453
- package/src/runtime/migrations/__tests__/vbundle-symlink-walker.test.ts +0 -319
- package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +0 -240
- package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +0 -473
- package/src/runtime/migrations/gcs-signed-url.ts +0 -190
- package/src/runtime/migrations/job-registry.ts +0 -297
- package/src/runtime/migrations/migration-transport.ts +0 -612
- package/src/runtime/migrations/migration-wizard.ts +0 -747
- package/src/runtime/migrations/origin-mode.ts +0 -42
- package/src/runtime/migrations/rebind-secrets-screen.ts +0 -412
- package/src/runtime/migrations/staged-import-path.ts +0 -116
- package/src/runtime/migrations/transfer-progress-screen.ts +0 -323
- package/src/runtime/migrations/validation-results-screen.ts +0 -469
- package/src/runtime/migrations/vbundle-builder.ts +0 -1382
- package/src/runtime/migrations/vbundle-import-analyzer.ts +0 -405
- package/src/runtime/migrations/vbundle-import-policy.ts +0 -241
- package/src/runtime/migrations/vbundle-importer.ts +0 -826
- package/src/runtime/migrations/vbundle-metadata-merge.ts +0 -113
- package/src/runtime/migrations/vbundle-streaming-importer.ts +0 -2844
- package/src/runtime/migrations/vbundle-streaming-validator.ts +0 -426
- package/src/runtime/migrations/vbundle-tar-stream.ts +0 -244
- package/src/runtime/migrations/vbundle-validator.ts +0 -724
- package/src/runtime/routes/__tests__/migration-cadet-metadata-reconcile.test.ts +0 -227
- package/src/runtime/routes/__tests__/migration-export-secrets-redacted.test.ts +0 -54
- package/src/runtime/routes/__tests__/migration-import-allowed-hosts-env.test.ts +0 -70
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +0 -257
- package/src/runtime/routes/background-wake-routes.ts +0 -387
- package/src/runtime/routes/migration-routes.ts +0 -2584
- package/src/tools/network/__tests__/cloud-search-proxy.test.ts +0 -282
- package/src/tools/network/cloud-search-proxy.ts +0 -211
- package/src/tts/__tests__/cadet-provider.test.ts +0 -253
- package/src/tts/providers/cadet-provider.ts +0 -244
- package/src/types/tar-stream.d.ts +0 -66
|
@@ -1,2584 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Route handlers for migration endpoints.
|
|
3
|
-
*
|
|
4
|
-
* POST /v1/migrations/validate — validate a .vbundle archive upload.
|
|
5
|
-
* POST /v1/migrations/export — generate and download a .vbundle archive.
|
|
6
|
-
* POST /v1/migrations/import-preflight — dry-run import analysis of a .vbundle archive.
|
|
7
|
-
* POST /v1/migrations/import — commit a .vbundle archive import to disk.
|
|
8
|
-
*
|
|
9
|
-
* Accepts raw binary body (Content-Type: application/octet-stream),
|
|
10
|
-
* multipart form data with a "file" field, or a JSON body of shape
|
|
11
|
-
* `{ "url": "<signed-gcs-url>" }` (import only) or `{ "path": "<staged>" }`
|
|
12
|
-
* (import and import-preflight). The path form streams a file the CLI
|
|
13
|
-
* staged under `${workspace}/.restore-staging/`. The URL form fetches the
|
|
14
|
-
* bundle from GCS and streams it through `streamCommitImport`.
|
|
15
|
-
* Returns structured validation results with is_valid flag and detailed
|
|
16
|
-
* error descriptions.
|
|
17
|
-
*/
|
|
18
|
-
|
|
19
|
-
import { createReadStream } from "node:fs";
|
|
20
|
-
import { hostname } from "node:os";
|
|
21
|
-
import { PassThrough, Readable } from "node:stream";
|
|
22
|
-
|
|
23
|
-
import { GatewayDebugExportIpcResponseSchema } from "@mycadet/gateway-client/gateway-ipc-contracts";
|
|
24
|
-
import { z } from "zod";
|
|
25
|
-
|
|
26
|
-
import { resolveCloudAssistantId } from "../../config/cloud-identity.js";
|
|
27
|
-
import { invalidateConfigCache } from "../../config/loader.js";
|
|
28
|
-
import { trackDaemonActivity } from "../../daemon/activity-trail.js";
|
|
29
|
-
import { getAssistantName } from "../../daemon/identity-helpers.js";
|
|
30
|
-
import { ipcCallPersistent } from "../../ipc/gateway-client.js";
|
|
31
|
-
import { runAsyncSqlite } from "../../persistence/db-async-query.js";
|
|
32
|
-
import {
|
|
33
|
-
getDb,
|
|
34
|
-
getLogsSqlite,
|
|
35
|
-
getMemorySqlite,
|
|
36
|
-
resetDb,
|
|
37
|
-
} from "../../persistence/db-connection.js";
|
|
38
|
-
import { validateMigrationState } from "../../persistence/migrations/validate-migration-state.js";
|
|
39
|
-
import { migrationSteps } from "../../persistence/steps.js";
|
|
40
|
-
import { credentialKey } from "../../security/credential-key.js";
|
|
41
|
-
import {
|
|
42
|
-
bulkSetSecureKeysAsync,
|
|
43
|
-
getSecureKeyAsync,
|
|
44
|
-
getSecureKeyResultAsync,
|
|
45
|
-
listSecureKeysAsync,
|
|
46
|
-
} from "../../security/secure-keys.js";
|
|
47
|
-
import {
|
|
48
|
-
getCredentialMetadata,
|
|
49
|
-
upsertCredentialMetadata,
|
|
50
|
-
} from "../../tools/credentials/metadata-store.js";
|
|
51
|
-
import { getWorkspaceDir, getWorkspaceHooksDir } from "../../util/layout.js";
|
|
52
|
-
import { getLogger } from "../../util/logger.js";
|
|
53
|
-
import { APP_VERSION } from "../../version.js";
|
|
54
|
-
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../assistant-scope.js";
|
|
55
|
-
import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
|
|
56
|
-
import {
|
|
57
|
-
validateGcsSignedUrl,
|
|
58
|
-
type ValidateGcsSignedUrlOptions,
|
|
59
|
-
} from "../migrations/gcs-signed-url.js";
|
|
60
|
-
import {
|
|
61
|
-
JobAlreadyInProgressError,
|
|
62
|
-
migrationJobs,
|
|
63
|
-
} from "../migrations/job-registry.js";
|
|
64
|
-
import { getOriginMode } from "../migrations/origin-mode.js";
|
|
65
|
-
import {
|
|
66
|
-
resolveStagedImportPath,
|
|
67
|
-
StagedImportPathError,
|
|
68
|
-
} from "../migrations/staged-import-path.js";
|
|
69
|
-
import type {
|
|
70
|
-
VBundleAssistantInfo,
|
|
71
|
-
VBundleCompatibility,
|
|
72
|
-
VBundleExportOptions,
|
|
73
|
-
VBundleOriginInfo,
|
|
74
|
-
} from "../migrations/vbundle-builder.js";
|
|
75
|
-
import { streamExportVBundle } from "../migrations/vbundle-builder.js";
|
|
76
|
-
import {
|
|
77
|
-
analyzeImport,
|
|
78
|
-
DefaultPathResolver,
|
|
79
|
-
} from "../migrations/vbundle-import-analyzer.js";
|
|
80
|
-
import {
|
|
81
|
-
evaluateRuntimeCompatibility,
|
|
82
|
-
formatRuntimeCompatibilityMessage,
|
|
83
|
-
type RuntimeCompatibility,
|
|
84
|
-
} from "../migrations/vbundle-import-policy.js";
|
|
85
|
-
import {
|
|
86
|
-
commitImport,
|
|
87
|
-
extractCredentialsFromBundle,
|
|
88
|
-
type ImportCommitReport,
|
|
89
|
-
type ImportCommitResult,
|
|
90
|
-
} from "../migrations/vbundle-importer.js";
|
|
91
|
-
import { streamCommitImport } from "../migrations/vbundle-streaming-importer.js";
|
|
92
|
-
import {
|
|
93
|
-
readAndValidateManifest,
|
|
94
|
-
StreamingValidationError,
|
|
95
|
-
} from "../migrations/vbundle-streaming-validator.js";
|
|
96
|
-
import { parseVBundleStream } from "../migrations/vbundle-tar-stream.js";
|
|
97
|
-
import { validateVBundle } from "../migrations/vbundle-validator.js";
|
|
98
|
-
import {
|
|
99
|
-
BadGatewayError,
|
|
100
|
-
BadRequestError,
|
|
101
|
-
InternalError,
|
|
102
|
-
NotFoundError,
|
|
103
|
-
RouteError,
|
|
104
|
-
UnprocessableEntityError,
|
|
105
|
-
} from "./errors.js";
|
|
106
|
-
import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
|
|
107
|
-
import { RouteResponse } from "./types.js";
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* CES account prefix for cloud-identity (`cadet:*`) credentials. Entries
|
|
111
|
-
* with an account that starts with this string are filtered out of any
|
|
112
|
-
* imported bundle so they don't overwrite the target's own Django-provisioned
|
|
113
|
-
* platform identity (most notably `assistant_api_key`).
|
|
114
|
-
*
|
|
115
|
-
* Derived from `credentialKey("cadet", "")` so the prefix automatically
|
|
116
|
-
* tracks the real CES account format — the literal string `"credential/cadet/"`.
|
|
117
|
-
*/
|
|
118
|
-
const PLATFORM_CREDENTIAL_PREFIX = credentialKey("cadet", "");
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Cadet credentials that the cloud runtime expects to see in CES.
|
|
122
|
-
* On a cloud assistant the cloud provisions these after hatch via
|
|
123
|
-
* `POST /v1/secrets`. A signed-in client may additionally (re)assert some of
|
|
124
|
-
* them on teleport / local→managed transfer. Either set of writes can race with
|
|
125
|
-
* the import — the CES write survives (separate volume), but the metadata
|
|
126
|
-
* upsert may be clobbered by the in-place clear / atomic swap. After every
|
|
127
|
-
* import we reconcile metadata.json against CES so any field CES already holds
|
|
128
|
-
* a value for gets a matching metadata entry.
|
|
129
|
-
*/
|
|
130
|
-
const CADET_CREDENTIAL_FIELDS = [
|
|
131
|
-
"assistant_api_key",
|
|
132
|
-
"webhook_secret",
|
|
133
|
-
] as const;
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Idempotent post-import reconciliation: for each cadet:* field, if CES
|
|
137
|
-
* has a value but metadata.json doesn't list it, upsert the entry. Pure
|
|
138
|
-
* add-only — never deletes anything. Safe to run whether or not Django's
|
|
139
|
-
* post-hatch provisioning has completed (missing CES values are skipped).
|
|
140
|
-
*
|
|
141
|
-
* Exported for direct unit-testing.
|
|
142
|
-
*/
|
|
143
|
-
export async function reconcileCadetMetadataFromCes(warningSink: {
|
|
144
|
-
warnings: string[];
|
|
145
|
-
}): Promise<void> {
|
|
146
|
-
for (const field of CADET_CREDENTIAL_FIELDS) {
|
|
147
|
-
try {
|
|
148
|
-
const value = await getSecureKeyAsync(credentialKey("cadet", field));
|
|
149
|
-
if (!value) {
|
|
150
|
-
continue;
|
|
151
|
-
}
|
|
152
|
-
if (getCredentialMetadata("cadet", field)) {
|
|
153
|
-
continue;
|
|
154
|
-
}
|
|
155
|
-
upsertCredentialMetadata("cadet", field, {});
|
|
156
|
-
log.info(
|
|
157
|
-
{ field },
|
|
158
|
-
"Reconciled cadet:* metadata entry from CES after import",
|
|
159
|
-
);
|
|
160
|
-
} catch (err) {
|
|
161
|
-
warningSink.warnings.push(
|
|
162
|
-
`Failed to reconcile cadet:${field} metadata: ${
|
|
163
|
-
err instanceof Error ? err.message : String(err)
|
|
164
|
-
}`,
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
const log = getLogger("migration-routes");
|
|
171
|
-
|
|
172
|
-
/**
|
|
173
|
-
* Flush each database file's WAL before an export copies them. The bundle
|
|
174
|
-
* walker includes `*.db` but skips `*.db-wal`, so committed frames still sitting
|
|
175
|
-
* in a WAL would be missing from the bundle.
|
|
176
|
-
*
|
|
177
|
-
* The main DB is flushed through `runAsyncSqlite` so the (potentially multi-GB)
|
|
178
|
-
* flush runs in a sqlite3 subprocess where available rather than stalling the
|
|
179
|
-
* event loop. The dedicated logs/memory connections are checkpointed in-process
|
|
180
|
-
* on their own handles — they are small and the daemon already holds them open.
|
|
181
|
-
* FULL (not TRUNCATE) writes committed frames back to the main file, which is
|
|
182
|
-
* all the copy needs.
|
|
183
|
-
*/
|
|
184
|
-
async function checkpointDbsForExport(): Promise<void> {
|
|
185
|
-
const mainResult = await runAsyncSqlite(
|
|
186
|
-
"PRAGMA wal_checkpoint(FULL)",
|
|
187
|
-
"export-checkpoint:wal-checkpoint-full",
|
|
188
|
-
);
|
|
189
|
-
if (!mainResult.ok) {
|
|
190
|
-
log.warn(
|
|
191
|
-
{ error: mainResult.error, backend: mainResult.backend, db: "main" },
|
|
192
|
-
"WAL checkpoint failed — exporting without checkpoint",
|
|
193
|
-
);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
for (const [label, sqlite] of [
|
|
197
|
-
["logs", getLogsSqlite()],
|
|
198
|
-
["memory", getMemorySqlite()],
|
|
199
|
-
// (no telemetry: migration 387 deletes that database file outright)
|
|
200
|
-
] as const) {
|
|
201
|
-
if (!sqlite) {
|
|
202
|
-
log.warn(
|
|
203
|
-
{ db: label },
|
|
204
|
-
"Dedicated database unavailable — exporting without checkpoint",
|
|
205
|
-
);
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
208
|
-
try {
|
|
209
|
-
sqlite.exec("PRAGMA wal_checkpoint(FULL)");
|
|
210
|
-
} catch (err) {
|
|
211
|
-
log.warn(
|
|
212
|
-
{ err, db: label },
|
|
213
|
-
"WAL checkpoint failed — exporting without checkpoint",
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
/**
|
|
220
|
-
* Fields the export pipeline must populate on the v1 manifest.
|
|
221
|
-
*
|
|
222
|
-
* Centralized so both the synchronous-bytes and async-to-gcs handlers
|
|
223
|
-
* compute the same values (and a future caller doesn't accidentally drift).
|
|
224
|
-
*/
|
|
225
|
-
interface ExportManifestInputs {
|
|
226
|
-
assistant: VBundleAssistantInfo;
|
|
227
|
-
origin: VBundleOriginInfo;
|
|
228
|
-
compatibility: VBundleCompatibility;
|
|
229
|
-
exportOptions: VBundleExportOptions;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/**
|
|
233
|
-
* Resolve the `assistant.id` for an export.
|
|
234
|
-
*
|
|
235
|
-
* In-memory identity (validate rehydration or secret-routes). The schema
|
|
236
|
-
* requires `id` to be non-empty, so we fall back to
|
|
237
|
-
* `DAEMON_INTERNAL_ASSISTANT_ID` rather than the empty string.
|
|
238
|
-
*/
|
|
239
|
-
async function resolveAssistantId(): Promise<string> {
|
|
240
|
-
const resolved = await resolveCloudAssistantId();
|
|
241
|
-
if (resolved) {
|
|
242
|
-
return resolved;
|
|
243
|
-
}
|
|
244
|
-
return DAEMON_INTERNAL_ASSISTANT_ID;
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/**
|
|
248
|
-
* Decide the truthful `secrets_redacted` flag for an export.
|
|
249
|
-
*
|
|
250
|
-
* The export entry points pass every collected credential through to the
|
|
251
|
-
* builder unfiltered, so the bundle is NOT redacted whenever any
|
|
252
|
-
* credentials made it in. Only flip to true when the credential list is
|
|
253
|
-
* empty AND every credential read succeeded — i.e. there genuinely are
|
|
254
|
-
* no secrets in the bundle.
|
|
255
|
-
*
|
|
256
|
-
* Two failure modes both force `false`:
|
|
257
|
-
* - `storeUnreachable`: the top-level `listSecureKeysAsync()` call
|
|
258
|
-
* failed, so we never even discovered which accounts exist.
|
|
259
|
-
* - `perAccountUnreachable`: the LIST call succeeded but one or more
|
|
260
|
-
* individual `getSecureKeyResultAsync(account)` reads returned
|
|
261
|
-
* `unreachable: true`. Those accounts were silently skipped from the
|
|
262
|
-
* `credentials` array, so a `credentialCount === 0` outcome could
|
|
263
|
-
* reflect "we couldn't read them" rather than "no secrets exist".
|
|
264
|
-
* Claiming a clean redaction in that case would be a lie.
|
|
265
|
-
*
|
|
266
|
-
* Managed-mode exports never reach this with credentials: both export
|
|
267
|
-
* handlers skip credential collection entirely when `origin.mode` is
|
|
268
|
-
* "managed" (the validator's cross-field refine rejects a cloud bundle
|
|
269
|
-
* with `secrets_redacted: false`, and teleport clients re-provision
|
|
270
|
-
* platform identity after a cloud→local import).
|
|
271
|
-
*/
|
|
272
|
-
export function computeSecretsRedacted(
|
|
273
|
-
credentialCount: number,
|
|
274
|
-
storeUnreachable: boolean,
|
|
275
|
-
perAccountUnreachable: boolean,
|
|
276
|
-
): boolean {
|
|
277
|
-
return credentialCount === 0 && !storeUnreachable && !perAccountUnreachable;
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
/**
|
|
281
|
-
* Compute the v1 manifest inputs that aren't tied to per-call options.
|
|
282
|
-
*
|
|
283
|
-
* `walkDirectoryForMetadata` skips `embedding-models`, `data/qdrant`,
|
|
284
|
-
* `signals`, and `deprecated` — `logs` is NOT in the skip list, so log
|
|
285
|
-
* files end up in `manifest.contents`. Browser state and memory vectors
|
|
286
|
-
* (qdrant) are skipped, so those flags are false.
|
|
287
|
-
*/
|
|
288
|
-
async function buildExportManifestInputs(): Promise<ExportManifestInputs> {
|
|
289
|
-
const assistantId = await resolveAssistantId();
|
|
290
|
-
const assistantName = getAssistantName() ?? "Assistant";
|
|
291
|
-
const originMode = await getOriginMode();
|
|
292
|
-
return {
|
|
293
|
-
assistant: {
|
|
294
|
-
id: assistantId,
|
|
295
|
-
name: assistantName,
|
|
296
|
-
runtime_version: APP_VERSION,
|
|
297
|
-
},
|
|
298
|
-
origin: {
|
|
299
|
-
mode: originMode,
|
|
300
|
-
hostname: hostname(),
|
|
301
|
-
},
|
|
302
|
-
compatibility: {
|
|
303
|
-
min_runtime_version: APP_VERSION,
|
|
304
|
-
max_runtime_version: null,
|
|
305
|
-
},
|
|
306
|
-
exportOptions: {
|
|
307
|
-
include_logs: true,
|
|
308
|
-
include_browser_state: false,
|
|
309
|
-
include_memory_vectors: false,
|
|
310
|
-
},
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
/**
|
|
315
|
-
* POST /v1/migrations/validate
|
|
316
|
-
*
|
|
317
|
-
* Validates a .vbundle archive. The file can be sent as:
|
|
318
|
-
* - Raw binary body with Content-Type: application/octet-stream
|
|
319
|
-
* - Multipart form data with a "file" field
|
|
320
|
-
*
|
|
321
|
-
* Returns:
|
|
322
|
-
* 200: { is_valid: true, manifest: { ... } }
|
|
323
|
-
* 200: { is_valid: false, errors: [{ code, message, path? }] }
|
|
324
|
-
* 400: Standard error envelope for missing/empty body
|
|
325
|
-
* 422: Standard error envelope for completely unparseable input
|
|
326
|
-
*/
|
|
327
|
-
export async function handleMigrationValidate({
|
|
328
|
-
rawBody,
|
|
329
|
-
headers,
|
|
330
|
-
}: RouteHandlerArgs) {
|
|
331
|
-
const fileData = await extractFileData(rawBody, headers);
|
|
332
|
-
|
|
333
|
-
try {
|
|
334
|
-
const result = validateVBundle(fileData);
|
|
335
|
-
|
|
336
|
-
return {
|
|
337
|
-
is_valid: result.is_valid,
|
|
338
|
-
errors: result.errors,
|
|
339
|
-
...(result.manifest ? { manifest: result.manifest } : {}),
|
|
340
|
-
};
|
|
341
|
-
} catch (err) {
|
|
342
|
-
log.error({ err }, "Unexpected error during vbundle validation");
|
|
343
|
-
throw new InternalError(
|
|
344
|
-
err instanceof Error ? err.message : "Unexpected validation error",
|
|
345
|
-
);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* POST /v1/migrations/export
|
|
351
|
-
*
|
|
352
|
-
* Exports the assistant's real data as a .vbundle archive. The archive
|
|
353
|
-
* contains the SQLite database (all conversations, messages, memory
|
|
354
|
-
* segments, embeddings) and the config file.
|
|
355
|
-
*
|
|
356
|
-
* Accepts an optional JSON body:
|
|
357
|
-
* { "description": "Human-readable export description" }
|
|
358
|
-
*
|
|
359
|
-
* Returns:
|
|
360
|
-
* 200: Binary .vbundle archive (Content-Type: application/octet-stream)
|
|
361
|
-
* with Content-Disposition header for download.
|
|
362
|
-
* 500: Standard error envelope for unexpected failures.
|
|
363
|
-
*
|
|
364
|
-
* Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
|
|
365
|
-
*/
|
|
366
|
-
export async function handleMigrationExport(
|
|
367
|
-
_args: RouteHandlerArgs,
|
|
368
|
-
): Promise<RouteResponse> {
|
|
369
|
-
// The legacy `description` field is no longer carried on the v1
|
|
370
|
-
// manifest. Older clients still POST it; we silently ignore it.
|
|
371
|
-
let cleanup: (() => Promise<void>) | undefined;
|
|
372
|
-
|
|
373
|
-
try {
|
|
374
|
-
const manifestInputs = await buildExportManifestInputs();
|
|
375
|
-
|
|
376
|
-
// Managed deployments never ship credentials in bundles — the manifest
|
|
377
|
-
// schema forbids it (`secrets_redacted` must be true when `origin.mode`
|
|
378
|
-
// is "managed"), and teleport clients re-provision platform identity
|
|
379
|
-
// after a cloud→local import. Redact instead of emitting a bundle
|
|
380
|
-
// the importer is guaranteed to reject.
|
|
381
|
-
let credentials: Array<{ account: string; value: string }> = [];
|
|
382
|
-
let secretsRedacted = true;
|
|
383
|
-
if (manifestInputs.origin.mode !== "managed") {
|
|
384
|
-
// Read all stored credentials to include in the export bundle
|
|
385
|
-
const credentialList = await listSecureKeysAsync();
|
|
386
|
-
credentials = [];
|
|
387
|
-
// Track per-account read failures separately from the top-level LIST
|
|
388
|
-
// failure. A single skipped account means we cannot truthfully claim
|
|
389
|
-
// the bundle is fully redacted — we don't know what we missed.
|
|
390
|
-
let perAccountUnreachable = false;
|
|
391
|
-
if (credentialList.unreachable) {
|
|
392
|
-
log.warn(
|
|
393
|
-
"Credential store is unreachable — export will not include credentials",
|
|
394
|
-
);
|
|
395
|
-
} else {
|
|
396
|
-
for (const account of credentialList.accounts) {
|
|
397
|
-
const result = await getSecureKeyResultAsync(account);
|
|
398
|
-
if (result.unreachable) {
|
|
399
|
-
perAccountUnreachable = true;
|
|
400
|
-
log.warn(
|
|
401
|
-
{ account },
|
|
402
|
-
"Credential store unreachable when reading credential — skipping",
|
|
403
|
-
);
|
|
404
|
-
} else if (result.value != null) {
|
|
405
|
-
credentials.push({ account, value: result.value });
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
secretsRedacted = computeSecretsRedacted(
|
|
410
|
-
credentials.length,
|
|
411
|
-
credentialList.unreachable,
|
|
412
|
-
perAccountUnreachable,
|
|
413
|
-
);
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
const result = await streamExportVBundle({
|
|
417
|
-
workspaceDir: getWorkspaceDir(),
|
|
418
|
-
...manifestInputs,
|
|
419
|
-
secretsRedacted,
|
|
420
|
-
credentials,
|
|
421
|
-
checkpoint: checkpointDbsForExport,
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
cleanup = result.cleanup;
|
|
425
|
-
const { tempPath, size, manifest } = result;
|
|
426
|
-
|
|
427
|
-
const timestamp = manifest.created_at.replace(/[:.]/g, "-");
|
|
428
|
-
const filename = `export-${timestamp}.vbundle`;
|
|
429
|
-
|
|
430
|
-
const fileStream = createReadStream(tempPath);
|
|
431
|
-
fileStream.on("close", () => {
|
|
432
|
-
cleanup?.();
|
|
433
|
-
cleanup = undefined;
|
|
434
|
-
});
|
|
435
|
-
|
|
436
|
-
const streamBody = Readable.toWeb(fileStream) as unknown as ReadableStream;
|
|
437
|
-
|
|
438
|
-
return new RouteResponse(streamBody, {
|
|
439
|
-
"Content-Type": "application/octet-stream",
|
|
440
|
-
"Content-Disposition": `attachment; filename="${filename}"`,
|
|
441
|
-
"Content-Length": String(size),
|
|
442
|
-
// `schema_version` is now an integer; clients that parse this header
|
|
443
|
-
// continue to work, but the value flips from "1.0" to "1".
|
|
444
|
-
"X-Vbundle-Schema-Version": String(manifest.schema_version),
|
|
445
|
-
// Header name preserved for cross-version client compat; populated
|
|
446
|
-
// from the renamed manifest `checksum` field.
|
|
447
|
-
"X-Vbundle-Manifest-Sha256": manifest.checksum,
|
|
448
|
-
"X-Vbundle-Credentials-Included": String(credentials.length),
|
|
449
|
-
});
|
|
450
|
-
} catch (err) {
|
|
451
|
-
await cleanup?.();
|
|
452
|
-
log.error({ err }, "Failed to build export bundle");
|
|
453
|
-
throw new InternalError(
|
|
454
|
-
err instanceof Error ? err.message : "Unexpected export error",
|
|
455
|
-
);
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
// ---------------------------------------------------------------------------
|
|
460
|
-
// POST /v1/migrations/export-to-gcs — async export streamed to a signed URL
|
|
461
|
-
// ---------------------------------------------------------------------------
|
|
462
|
-
|
|
463
|
-
/** 60 minutes — matches the URL-body import fetch deadline. */
|
|
464
|
-
const EXPORT_TO_GCS_PUT_TIMEOUT_MS = 60 * 60 * 1000;
|
|
465
|
-
|
|
466
|
-
const MigrationExportToGcsBody = z.object({
|
|
467
|
-
upload_url: z.string().url(),
|
|
468
|
-
description: z.string().optional(),
|
|
469
|
-
/**
|
|
470
|
-
* `debug`: a bundle for Cadet staff to inspect. Adds the gateway's
|
|
471
|
-
* database and recent logs under `gateway/`. `migration` is the default.
|
|
472
|
-
*/
|
|
473
|
-
profile: z.enum(["migration", "debug"]).default("migration"),
|
|
474
|
-
});
|
|
475
|
-
|
|
476
|
-
/** Archive path of the gateway's contribution to a debug bundle. */
|
|
477
|
-
export const GATEWAY_DEBUG_EXPORT_PATH = "gateway/export.tar.gz";
|
|
478
|
-
|
|
479
|
-
/**
|
|
480
|
-
* Ask the gateway for its debug archive over the local socket. The daemon
|
|
481
|
-
* cannot read the gateway's files itself: in local mode they sit in the
|
|
482
|
-
* protected directory beside the encryption keys, and in Docker on a volume
|
|
483
|
-
* the daemon does not mount.
|
|
484
|
-
*/
|
|
485
|
-
async function collectGatewayDebugExport(): Promise<{
|
|
486
|
-
archivePath: string;
|
|
487
|
-
data: Uint8Array;
|
|
488
|
-
}> {
|
|
489
|
-
const raw = await ipcCallPersistent(
|
|
490
|
-
"gateway_debug_export",
|
|
491
|
-
{},
|
|
492
|
-
GATEWAY_DEBUG_EXPORT_TIMEOUT_MS,
|
|
493
|
-
);
|
|
494
|
-
const parsed = GatewayDebugExportIpcResponseSchema.parse(raw);
|
|
495
|
-
return {
|
|
496
|
-
archivePath: GATEWAY_DEBUG_EXPORT_PATH,
|
|
497
|
-
data: new Uint8Array(Buffer.from(parsed.archive_base64, "base64")),
|
|
498
|
-
};
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
const GATEWAY_DEBUG_EXPORT_TIMEOUT_MS = 120_000;
|
|
502
|
-
|
|
503
|
-
/**
|
|
504
|
-
* Collected credentials plus warning markers if the credential store was
|
|
505
|
-
* unreachable. The caller surfaces the warning in logs; production callers
|
|
506
|
-
* fail closed on errors (a thrown exception → 500) to avoid shipping a
|
|
507
|
-
* bundle with partial credentials. An unreachable store is NOT an error —
|
|
508
|
-
* `handleMigrationExport` treats that case as "export without credentials".
|
|
509
|
-
*
|
|
510
|
-
* - `unreachable`: the top-level `listSecureKeysAsync()` call failed.
|
|
511
|
-
* - `perAccountUnreachable`: the LIST succeeded but one or more individual
|
|
512
|
-
* `getSecureKeyResultAsync(account)` calls returned `unreachable: true`.
|
|
513
|
-
* Those accounts were silently skipped from `credentials`, so the count
|
|
514
|
-
* here understates reality. The flag is what tells `computeSecretsRedacted`
|
|
515
|
-
* it cannot claim a clean redaction.
|
|
516
|
-
*/
|
|
517
|
-
interface CollectedCredentials {
|
|
518
|
-
credentials: Array<{ account: string; value: string }>;
|
|
519
|
-
unreachable: boolean;
|
|
520
|
-
perAccountUnreachable: boolean;
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
/**
|
|
524
|
-
* Mirror of the credential-collection block inside `handleMigrationExport`.
|
|
525
|
-
* Factored out so the new async export-to-gcs handler can share the exact
|
|
526
|
-
* same behavior. Throws if the credential store raises an unexpected error —
|
|
527
|
-
* the caller translates that into a 500 (fail closed on credential errors).
|
|
528
|
-
*/
|
|
529
|
-
async function collectExportCredentials(): Promise<CollectedCredentials> {
|
|
530
|
-
const credentialList = await listSecureKeysAsync();
|
|
531
|
-
if (credentialList.unreachable) {
|
|
532
|
-
log.warn(
|
|
533
|
-
"Credential store is unreachable — export will not include credentials",
|
|
534
|
-
);
|
|
535
|
-
return {
|
|
536
|
-
credentials: [],
|
|
537
|
-
unreachable: true,
|
|
538
|
-
perAccountUnreachable: false,
|
|
539
|
-
};
|
|
540
|
-
}
|
|
541
|
-
const credentials: Array<{ account: string; value: string }> = [];
|
|
542
|
-
let perAccountUnreachable = false;
|
|
543
|
-
for (const account of credentialList.accounts) {
|
|
544
|
-
const result = await getSecureKeyResultAsync(account);
|
|
545
|
-
if (result.unreachable) {
|
|
546
|
-
perAccountUnreachable = true;
|
|
547
|
-
log.warn(
|
|
548
|
-
{ account },
|
|
549
|
-
"Credential store unreachable when reading credential — skipping",
|
|
550
|
-
);
|
|
551
|
-
} else if (result.value != null) {
|
|
552
|
-
credentials.push({ account, value: result.value });
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
return { credentials, unreachable: false, perAccountUnreachable };
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
/**
|
|
559
|
-
* POST /v1/migrations/export-to-gcs
|
|
560
|
-
*
|
|
561
|
-
* Starts an async export job that streams a freshly-built .vbundle archive
|
|
562
|
-
* to a GCS signed PUT URL. Returns `202 Accepted` with a `job_id` the caller
|
|
563
|
-
* can poll via the job-status endpoint; the bundle upload runs in the
|
|
564
|
-
* background via `migrationJobs`.
|
|
565
|
-
*
|
|
566
|
-
* Request body (JSON):
|
|
567
|
-
* { upload_url: string, description?: string }
|
|
568
|
-
*
|
|
569
|
-
* Responses:
|
|
570
|
-
* 202: { job_id, status: "pending", type: "export" }
|
|
571
|
-
* 400: { error: { code: "invalid_upload_url", reason } } — URL failed
|
|
572
|
-
* `validateGcsSignedUrl` (scheme/host/signature/traversal).
|
|
573
|
-
* 409: { error: { code: "export_in_progress", job_id } } — another export
|
|
574
|
-
* job is already pending or running.
|
|
575
|
-
* 500: Standard error envelope for credential-collection failures or
|
|
576
|
-
* other unexpected errors before the job is enqueued.
|
|
577
|
-
*
|
|
578
|
-
* Terminal job state (surfaced via the job-status endpoint once poll lands):
|
|
579
|
-
* result: { size, sha256, schemaVersion, credentialsIncluded }
|
|
580
|
-
* error.code = "upload_failed" with `upstreamStatus` on non-2xx PUT
|
|
581
|
-
* error.code = "fetch_failed" on transport errors from the PUT itself.
|
|
582
|
-
*
|
|
583
|
-
* Auth: settings.write scope (matches `migrations/export`).
|
|
584
|
-
*/
|
|
585
|
-
export async function handleMigrationExportToGcs({ body }: RouteHandlerArgs) {
|
|
586
|
-
// ── 1. Parse JSON body ────────────────────────────────────────────────
|
|
587
|
-
const parsed = MigrationExportToGcsBody.safeParse(body);
|
|
588
|
-
if (!parsed.success) {
|
|
589
|
-
throw new BadRequestError(
|
|
590
|
-
"Request body must be { upload_url: string, description?: string, profile?: 'migration' | 'debug' } with a valid URL",
|
|
591
|
-
);
|
|
592
|
-
}
|
|
593
|
-
const isDebugProfile = parsed.data.profile === "debug";
|
|
594
|
-
|
|
595
|
-
// ── 2. Validate the upload URL. Never log `parsed.data.upload_url`.
|
|
596
|
-
const validated = validateGcsSignedUrl(
|
|
597
|
-
parsed.data.upload_url,
|
|
598
|
-
exportValidatorOptions(),
|
|
599
|
-
);
|
|
600
|
-
if (!validated.ok) {
|
|
601
|
-
log.warn(
|
|
602
|
-
{ reason: validated.reason },
|
|
603
|
-
"Rejected migration export-to-gcs upload URL",
|
|
604
|
-
);
|
|
605
|
-
throw new RouteError(
|
|
606
|
-
`Invalid upload URL: ${validated.reason}`,
|
|
607
|
-
"invalid_upload_url",
|
|
608
|
-
400,
|
|
609
|
-
);
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
log.info(
|
|
613
|
-
{ host: validated.host, path: validated.path },
|
|
614
|
-
"migration export to GCS starting",
|
|
615
|
-
);
|
|
616
|
-
|
|
617
|
-
const uploadUrl = parsed.data.upload_url;
|
|
618
|
-
|
|
619
|
-
// Compute the v1 manifest inputs once outside the async job runner so we
|
|
620
|
-
// surface failures (e.g. credential-store probe) as a synchronous 500
|
|
621
|
-
// before the caller starts polling.
|
|
622
|
-
let manifestInputs: ExportManifestInputs;
|
|
623
|
-
try {
|
|
624
|
-
manifestInputs = await buildExportManifestInputs();
|
|
625
|
-
} catch (err) {
|
|
626
|
-
log.error({ err }, "Failed to assemble export manifest inputs");
|
|
627
|
-
throw new InternalError(
|
|
628
|
-
err instanceof Error
|
|
629
|
-
? err.message
|
|
630
|
-
: "Failed to assemble export manifest inputs",
|
|
631
|
-
);
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
// ── 3. Collect credentials up front. Fail closed → 500. Managed
|
|
635
|
-
// deployments never ship credentials in bundles — the manifest schema
|
|
636
|
-
// forbids it (`secrets_redacted` must be true when `origin.mode` is
|
|
637
|
-
// "managed"), and teleport clients re-provision platform identity after
|
|
638
|
-
// a cloud→local import — so redact instead of emitting a bundle the
|
|
639
|
-
// importer is guaranteed to reject.
|
|
640
|
-
//
|
|
641
|
-
// A debug bundle goes to Cadet staff, so it never carries credentials
|
|
642
|
-
// either, whatever the deployment mode.
|
|
643
|
-
let collected: CollectedCredentials;
|
|
644
|
-
if (manifestInputs.origin.mode === "managed" || isDebugProfile) {
|
|
645
|
-
collected = {
|
|
646
|
-
credentials: [],
|
|
647
|
-
unreachable: false,
|
|
648
|
-
perAccountUnreachable: false,
|
|
649
|
-
};
|
|
650
|
-
} else {
|
|
651
|
-
try {
|
|
652
|
-
collected = await collectExportCredentials();
|
|
653
|
-
} catch (err) {
|
|
654
|
-
log.error({ err }, "Failed to collect credentials for export-to-gcs");
|
|
655
|
-
throw new InternalError(
|
|
656
|
-
err instanceof Error ? err.message : "Failed to collect credentials",
|
|
657
|
-
);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
const secretsRedacted = computeSecretsRedacted(
|
|
662
|
-
collected.credentials.length,
|
|
663
|
-
collected.unreachable,
|
|
664
|
-
collected.perAccountUnreachable,
|
|
665
|
-
);
|
|
666
|
-
|
|
667
|
-
if (isDebugProfile) {
|
|
668
|
-
manifestInputs = {
|
|
669
|
-
...manifestInputs,
|
|
670
|
-
exportOptions: { ...manifestInputs.exportOptions, include_gateway: true },
|
|
671
|
-
};
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// ── 4. Enqueue the job. The runner captures the collected credentials.
|
|
675
|
-
let job;
|
|
676
|
-
try {
|
|
677
|
-
job = migrationJobs.startJob("export", async () => {
|
|
678
|
-
let cleanup: (() => Promise<void>) | undefined;
|
|
679
|
-
try {
|
|
680
|
-
// The gateway snapshot can take a while on a large database, so it
|
|
681
|
-
// runs inside the job (after the export slot is held) rather than
|
|
682
|
-
// before the 202. A debug bundle without the gateway's data is not
|
|
683
|
-
// what staff asked for, so a failure fails the job and nothing is
|
|
684
|
-
// uploaded.
|
|
685
|
-
let extraFiles: Array<{ archivePath: string; data: Uint8Array }> = [];
|
|
686
|
-
if (isDebugProfile) {
|
|
687
|
-
try {
|
|
688
|
-
extraFiles = [await collectGatewayDebugExport()];
|
|
689
|
-
} catch (err) {
|
|
690
|
-
log.error({ err }, "Failed to collect the gateway's debug export");
|
|
691
|
-
const wrapped = new Error(
|
|
692
|
-
"The gateway did not provide its debug export; the bundle was not sent.",
|
|
693
|
-
);
|
|
694
|
-
(wrapped as { code?: string }).code = "gateway_debug_export_failed";
|
|
695
|
-
throw wrapped;
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
const result = await streamExportVBundle({
|
|
700
|
-
workspaceDir: getWorkspaceDir(),
|
|
701
|
-
...manifestInputs,
|
|
702
|
-
secretsRedacted,
|
|
703
|
-
credentials: collected.credentials,
|
|
704
|
-
extraFiles,
|
|
705
|
-
checkpoint: checkpointDbsForExport,
|
|
706
|
-
});
|
|
707
|
-
|
|
708
|
-
cleanup = result.cleanup;
|
|
709
|
-
const { tempPath, size, manifest } = result;
|
|
710
|
-
|
|
711
|
-
// Stream the temp file to GCS via PUT. Using Node's ReadableStream
|
|
712
|
-
// bridge keeps peak memory bounded — we do NOT load the archive
|
|
713
|
-
// into memory.
|
|
714
|
-
const fileStream = createReadStream(tempPath);
|
|
715
|
-
const webBody = Readable.toWeb(
|
|
716
|
-
fileStream,
|
|
717
|
-
) as unknown as ReadableStream<Uint8Array>;
|
|
718
|
-
|
|
719
|
-
let response: Response;
|
|
720
|
-
try {
|
|
721
|
-
response = await fetch(uploadUrl, {
|
|
722
|
-
method: "PUT",
|
|
723
|
-
body: webBody,
|
|
724
|
-
// `duplex: "half"` is required when sending a streaming body
|
|
725
|
-
// via fetch in Node/Bun — without it the cloud rejects the
|
|
726
|
-
// request as "duplex option is required when body is a
|
|
727
|
-
// ReadableStream".
|
|
728
|
-
duplex: "half",
|
|
729
|
-
// `validateGcsSignedUrl` only vets the initial URL. If the
|
|
730
|
-
// upstream responds with a 3xx, default fetch would follow
|
|
731
|
-
// the redirect and PUT bytes to an attacker-controlled host.
|
|
732
|
-
// Refuse redirects so the signed URL's origin is the only
|
|
733
|
-
// destination for the export archive.
|
|
734
|
-
redirect: "error",
|
|
735
|
-
headers: {
|
|
736
|
-
"Content-Type": "application/octet-stream",
|
|
737
|
-
"Content-Length": String(size),
|
|
738
|
-
},
|
|
739
|
-
signal: AbortSignal.timeout(EXPORT_TO_GCS_PUT_TIMEOUT_MS),
|
|
740
|
-
} as RequestInit & { duplex: "half" });
|
|
741
|
-
} catch (err) {
|
|
742
|
-
// Transport-level fetch failures (DNS, reset, abort) — tag them
|
|
743
|
-
// with the fetch-body marker so the registry maps them to
|
|
744
|
-
// `error.code = "fetch_failed"` and logs stay consistent with
|
|
745
|
-
// the import URL path.
|
|
746
|
-
const wrapped = err instanceof Error ? err : new Error(String(err));
|
|
747
|
-
tagFetchBodyError(wrapped as NodeJS.ErrnoException);
|
|
748
|
-
(wrapped as { code?: string }).code = "fetch_failed";
|
|
749
|
-
throw wrapped;
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
if (!response.ok) {
|
|
753
|
-
// Drain so the socket is released promptly. Ignore drain errors.
|
|
754
|
-
try {
|
|
755
|
-
await response.body?.cancel();
|
|
756
|
-
} catch {
|
|
757
|
-
/* best-effort */
|
|
758
|
-
}
|
|
759
|
-
const uploadErr = new Error(
|
|
760
|
-
`Upload to GCS failed with status ${response.status}`,
|
|
761
|
-
);
|
|
762
|
-
(uploadErr as { code?: string }).code = "upload_failed";
|
|
763
|
-
(uploadErr as { upstreamStatus?: number }).upstreamStatus =
|
|
764
|
-
response.status;
|
|
765
|
-
throw uploadErr;
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
return {
|
|
769
|
-
size,
|
|
770
|
-
sha256: manifest.checksum,
|
|
771
|
-
schemaVersion: manifest.schema_version,
|
|
772
|
-
credentialsIncluded: collected.credentials.length,
|
|
773
|
-
};
|
|
774
|
-
} finally {
|
|
775
|
-
// Mirror the raw-bytes export cleanup pattern: the stream's
|
|
776
|
-
// `close` listener is the happy-path cleanup in that handler, but
|
|
777
|
-
// here we keep everything in the async block, so a finally is the
|
|
778
|
-
// right place to evict the temp file regardless of outcome.
|
|
779
|
-
if (cleanup) {
|
|
780
|
-
try {
|
|
781
|
-
await cleanup();
|
|
782
|
-
} catch (err) {
|
|
783
|
-
log.warn({ err }, "Failed to clean up export-to-gcs temp file");
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
}
|
|
787
|
-
});
|
|
788
|
-
} catch (err) {
|
|
789
|
-
if (err instanceof JobAlreadyInProgressError) {
|
|
790
|
-
throw new RouteError(
|
|
791
|
-
`Export already in progress: ${err.existingJobId}`,
|
|
792
|
-
"export_in_progress",
|
|
793
|
-
409,
|
|
794
|
-
);
|
|
795
|
-
}
|
|
796
|
-
log.error({ err }, "Unexpected error while enqueueing export-to-gcs job");
|
|
797
|
-
throw new InternalError(
|
|
798
|
-
err instanceof Error ? err.message : "Unexpected export-to-gcs error",
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
return {
|
|
803
|
-
job_id: job.id,
|
|
804
|
-
status: "pending" as const,
|
|
805
|
-
type: "export" as const,
|
|
806
|
-
};
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
/**
|
|
810
|
-
* Extract file data from a request body, supporting both raw binary
|
|
811
|
-
* and multipart form data uploads.
|
|
812
|
-
*
|
|
813
|
-
* Shared between validate and import-preflight handlers.
|
|
814
|
-
*/
|
|
815
|
-
async function extractFileData(
|
|
816
|
-
rawBody: Uint8Array | undefined,
|
|
817
|
-
headers: Record<string, string> | undefined,
|
|
818
|
-
): Promise<Uint8Array> {
|
|
819
|
-
const contentType = headers?.["content-type"] ?? "";
|
|
820
|
-
|
|
821
|
-
if (contentType.includes("multipart/form-data")) {
|
|
822
|
-
if (!rawBody) {
|
|
823
|
-
throw new BadRequestError("Request body is empty");
|
|
824
|
-
}
|
|
825
|
-
try {
|
|
826
|
-
const syntheticReq = new Request("http://localhost", {
|
|
827
|
-
method: "POST",
|
|
828
|
-
headers: { "content-type": contentType },
|
|
829
|
-
body: rawBody.buffer as ArrayBuffer,
|
|
830
|
-
});
|
|
831
|
-
const formData = await syntheticReq.formData();
|
|
832
|
-
const file = formData.get("file");
|
|
833
|
-
if (!file || !(file instanceof Blob)) {
|
|
834
|
-
throw new BadRequestError('Multipart upload requires a "file" field');
|
|
835
|
-
}
|
|
836
|
-
return new Uint8Array(await file.arrayBuffer());
|
|
837
|
-
} catch (err) {
|
|
838
|
-
if (err instanceof BadRequestError) {
|
|
839
|
-
throw err;
|
|
840
|
-
}
|
|
841
|
-
log.error({ err }, "Failed to parse multipart form data");
|
|
842
|
-
throw new BadRequestError("Invalid multipart form data");
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
// Raw binary body — already provided as rawBody by the adapter
|
|
847
|
-
if (!rawBody || rawBody.length === 0) {
|
|
848
|
-
throw new BadRequestError(
|
|
849
|
-
"Request body is empty — a .vbundle file is required",
|
|
850
|
-
);
|
|
851
|
-
}
|
|
852
|
-
return rawBody;
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
/**
|
|
856
|
-
* POST /v1/migrations/import-preflight
|
|
857
|
-
*
|
|
858
|
-
* Dry-run import analysis. Accepts a .vbundle archive upload, validates it,
|
|
859
|
-
* and returns a detailed report of what would change if the bundle were
|
|
860
|
-
* actually imported — without modifying any data on disk.
|
|
861
|
-
*
|
|
862
|
-
* The file can be sent as:
|
|
863
|
-
* - Raw binary body with Content-Type: application/octet-stream
|
|
864
|
-
* - Multipart form data with a "file" field
|
|
865
|
-
* - JSON body `{ "path": "<staged-relative-or-absolute>" }` pointing at a
|
|
866
|
-
* `.vbundle` the caller placed under `${workspace}/.restore-staging/`
|
|
867
|
-
*
|
|
868
|
-
* Returns:
|
|
869
|
-
* 200: {
|
|
870
|
-
* can_import: boolean,
|
|
871
|
-
* summary: { total_files, files_to_create, files_to_overwrite, files_unchanged },
|
|
872
|
-
* files: [{ path, action, bundle_size, current_size, bundle_sha256, current_sha256 }],
|
|
873
|
-
* conflicts: [{ code, message, path? }],
|
|
874
|
-
* manifest: { ... }
|
|
875
|
-
* }
|
|
876
|
-
* 200: { can_import: false, validation: { is_valid: false, errors: [...] } }
|
|
877
|
-
* (when the bundle itself is invalid)
|
|
878
|
-
* 400: Standard error envelope for missing/empty body
|
|
879
|
-
* 500: Standard error envelope for unexpected failures
|
|
880
|
-
*
|
|
881
|
-
* Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
|
|
882
|
-
*/
|
|
883
|
-
export async function handleMigrationImportPreflight({
|
|
884
|
-
rawBody,
|
|
885
|
-
headers,
|
|
886
|
-
body,
|
|
887
|
-
}: RouteHandlerArgs) {
|
|
888
|
-
const contentType = headers?.["content-type"] ?? "";
|
|
889
|
-
if (contentType.includes("application/json")) {
|
|
890
|
-
return handleMigrationPreflightFromPath(body);
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
const fileData = await extractFileData(rawBody, headers);
|
|
894
|
-
|
|
895
|
-
try {
|
|
896
|
-
const validationResult = validateVBundle(fileData);
|
|
897
|
-
|
|
898
|
-
if (!validationResult.is_valid || !validationResult.manifest) {
|
|
899
|
-
return {
|
|
900
|
-
can_import: false,
|
|
901
|
-
validation: {
|
|
902
|
-
is_valid: false,
|
|
903
|
-
errors: validationResult.errors,
|
|
904
|
-
},
|
|
905
|
-
};
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
const pathResolver = new DefaultPathResolver(
|
|
909
|
-
getWorkspaceDir(),
|
|
910
|
-
getWorkspaceHooksDir(),
|
|
911
|
-
);
|
|
912
|
-
|
|
913
|
-
return analyzeImport({
|
|
914
|
-
manifest: validationResult.manifest,
|
|
915
|
-
pathResolver,
|
|
916
|
-
});
|
|
917
|
-
} catch (err) {
|
|
918
|
-
log.error({ err }, "Unexpected error during import preflight analysis");
|
|
919
|
-
throw new InternalError(
|
|
920
|
-
err instanceof Error ? err.message : "Unexpected import preflight error",
|
|
921
|
-
);
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
|
|
925
|
-
/**
|
|
926
|
-
* POST /v1/migrations/import
|
|
927
|
-
*
|
|
928
|
-
* Commits a .vbundle archive import to disk. This is a destructive operation
|
|
929
|
-
* that writes bundle files to their target locations, replacing existing data.
|
|
930
|
-
*
|
|
931
|
-
* The import process:
|
|
932
|
-
* 1. Validates the bundle (validation before any state mutation)
|
|
933
|
-
* 2. Extracts files from the archive
|
|
934
|
-
* 3. Backs up existing files before overwriting
|
|
935
|
-
* 4. Writes bundle files to disk
|
|
936
|
-
* 5. Verifies post-write integrity (SHA-256 check)
|
|
937
|
-
* 6. Returns a detailed report of what was imported
|
|
938
|
-
*
|
|
939
|
-
* The bundle can be supplied as:
|
|
940
|
-
* - Raw binary body with Content-Type: application/octet-stream
|
|
941
|
-
* - Multipart form data with a "file" field
|
|
942
|
-
* - JSON body `{ "url": "<signed-gcs-url>" }` (Content-Type:
|
|
943
|
-
* application/json). The daemon fetches and streams the archive
|
|
944
|
-
* through `streamCommitImport`, so peak memory stays bounded by a
|
|
945
|
-
* single tar entry rather than bundle size.
|
|
946
|
-
* - JSON body `{ "path": "<staged>" }` pointing at a `.vbundle` the
|
|
947
|
-
* caller placed under `${workspace}/.restore-staging/`. The daemon
|
|
948
|
-
* streams that file through `streamCommitImport`.
|
|
949
|
-
*
|
|
950
|
-
* Returns:
|
|
951
|
-
* 200: {
|
|
952
|
-
* success: true,
|
|
953
|
-
* summary: { total_files, files_created, files_overwritten, files_skipped, backups_created },
|
|
954
|
-
* files: [{ path, disk_path, action, size, sha256, backup_path }],
|
|
955
|
-
* manifest: { ... },
|
|
956
|
-
* warnings: [...]
|
|
957
|
-
* }
|
|
958
|
-
* 200: { success: false, reason: "validation_failed", errors: [...] }
|
|
959
|
-
* 400: Standard error envelope for missing/empty body or malformed URL
|
|
960
|
-
* 500: Standard error envelope for unexpected failures
|
|
961
|
-
* 502: { success: false, reason: "fetch_failed", upstream_status?: number }
|
|
962
|
-
* (URL path only — upstream GCS fetch failed)
|
|
963
|
-
*
|
|
964
|
-
* Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
|
|
965
|
-
*/
|
|
966
|
-
export async function handleMigrationImport(
|
|
967
|
-
args: RouteHandlerArgs,
|
|
968
|
-
): Promise<unknown> {
|
|
969
|
-
const { body, rawBody, headers } = args;
|
|
970
|
-
// JSON body is either a signed URL fetch or a staged local-file stream.
|
|
971
|
-
const contentType = headers?.["content-type"] ?? "";
|
|
972
|
-
if (contentType.includes("application/json")) {
|
|
973
|
-
return handleMigrationImportFromJson(body);
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
const fileData = await extractFileData(rawBody, headers);
|
|
977
|
-
|
|
978
|
-
try {
|
|
979
|
-
// Validate the bundle before closing the DB to avoid an unnecessary
|
|
980
|
-
// close/reopen cycle when the bundle is invalid. Pass the validated
|
|
981
|
-
// manifest and entries to commitImport so it skips re-validation
|
|
982
|
-
// (avoids holding two copies of decompressed data in memory).
|
|
983
|
-
const validation = validateVBundle(fileData);
|
|
984
|
-
if (!validation.is_valid) {
|
|
985
|
-
return {
|
|
986
|
-
success: false,
|
|
987
|
-
reason: "validation_failed",
|
|
988
|
-
errors: validation.errors,
|
|
989
|
-
};
|
|
990
|
-
}
|
|
991
|
-
|
|
992
|
-
// Pre-check runtime-version compat before the DB close/reopen cycle.
|
|
993
|
-
// commitImport runs the same gate as defense-in-depth for callers that
|
|
994
|
-
// don't pre-check; we run it here too so an incompatible bundle short-
|
|
995
|
-
// circuits before resetDb().
|
|
996
|
-
const compatResult = evaluateRuntimeCompatibility(
|
|
997
|
-
validation.manifest!.compatibility,
|
|
998
|
-
APP_VERSION,
|
|
999
|
-
);
|
|
1000
|
-
if (!compatResult.ok) {
|
|
1001
|
-
throwImportCommitFailure({
|
|
1002
|
-
ok: false,
|
|
1003
|
-
reason: "version_incompatible",
|
|
1004
|
-
bundle_compat: compatResult.bundle_compat,
|
|
1005
|
-
runtime_version: compatResult.runtime_version,
|
|
1006
|
-
});
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
const pathResolver = new DefaultPathResolver(
|
|
1010
|
-
getWorkspaceDir(),
|
|
1011
|
-
getWorkspaceHooksDir(),
|
|
1012
|
-
);
|
|
1013
|
-
|
|
1014
|
-
// Close the live SQLite connection before overwriting assistant.db on disk.
|
|
1015
|
-
// The singleton will be lazily reopened on the next getDb() call.
|
|
1016
|
-
resetDb();
|
|
1017
|
-
|
|
1018
|
-
const result = commitImport({
|
|
1019
|
-
archiveData: fileData,
|
|
1020
|
-
pathResolver,
|
|
1021
|
-
preValidatedManifest: validation.manifest,
|
|
1022
|
-
preValidatedEntries: validation.entries,
|
|
1023
|
-
workspaceDir: getWorkspaceDir(),
|
|
1024
|
-
});
|
|
1025
|
-
|
|
1026
|
-
if (!result.ok) {
|
|
1027
|
-
throwImportCommitFailure(result);
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
// Import credentials from the bundle into CES (non-blocking — failures
|
|
1031
|
-
// are logged as warnings but do not fail the overall import).
|
|
1032
|
-
let credentialsImported: CredentialImportSummary | undefined;
|
|
1033
|
-
|
|
1034
|
-
if (validation.entries) {
|
|
1035
|
-
const bundleCredentials = extractCredentialsFromBundle(
|
|
1036
|
-
validation.entries,
|
|
1037
|
-
validation.manifest!,
|
|
1038
|
-
);
|
|
1039
|
-
credentialsImported = await importBundleCredentialsIntoCes(
|
|
1040
|
-
bundleCredentials,
|
|
1041
|
-
result.report,
|
|
1042
|
-
);
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
// Reconcile cadet:* metadata against CES so the gateway's
|
|
1046
|
-
// readServiceCredentials can still find platform identity values even
|
|
1047
|
-
// if Django's post-hatch provisioning raced with the import.
|
|
1048
|
-
await reconcileCadetMetadataFromCes(result.report);
|
|
1049
|
-
|
|
1050
|
-
// Invalidate in-process config cache so imported settings.json takes effect
|
|
1051
|
-
invalidateConfigCache();
|
|
1052
|
-
|
|
1053
|
-
// Check whether the imported database contains migration checkpoints from
|
|
1054
|
-
// a newer version. This is non-blocking — the import has already
|
|
1055
|
-
// succeeded — but we surface a warning so the caller knows some data may
|
|
1056
|
-
// not be fully compatible with this daemon's schema.
|
|
1057
|
-
appendNewerMigrationWarningsIfAny(result.report);
|
|
1058
|
-
|
|
1059
|
-
return importCommitSuccessResult(result.report, credentialsImported);
|
|
1060
|
-
} catch (err) {
|
|
1061
|
-
// Preserve typed RouteError instances (e.g. UnprocessableEntityError for
|
|
1062
|
-
// version_incompatible, BadRequestError for validation_failed) — only
|
|
1063
|
-
// wrap genuinely unexpected errors as 500 InternalError.
|
|
1064
|
-
if (err instanceof RouteError) {
|
|
1065
|
-
throw err;
|
|
1066
|
-
}
|
|
1067
|
-
log.error({ err }, "Unexpected error during import commit");
|
|
1068
|
-
throw new InternalError(
|
|
1069
|
-
err instanceof Error ? err.message : "Unexpected import error",
|
|
1070
|
-
);
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
// ---------------------------------------------------------------------------
|
|
1075
|
-
// GCS URL import pipeline — shared by the URL-body branch of
|
|
1076
|
-
// POST /v1/migrations/import and POST /v1/migrations/import-from-gcs.
|
|
1077
|
-
// ---------------------------------------------------------------------------
|
|
1078
|
-
|
|
1079
|
-
/** 60 minutes — matches the gateway's upstream fetch deadline. */
|
|
1080
|
-
const URL_FETCH_TIMEOUT_MS = 60 * 60 * 1000;
|
|
1081
|
-
|
|
1082
|
-
const MigrationImportUrlBody = z.object({ url: z.string().min(1) });
|
|
1083
|
-
|
|
1084
|
-
const MigrationImportPathBody = z.object({ path: z.string().min(1) });
|
|
1085
|
-
|
|
1086
|
-
const MigrationImportFromGcsBody = z.object({ bundle_url: z.string().url() });
|
|
1087
|
-
|
|
1088
|
-
/**
|
|
1089
|
-
* Marker attached to errors that originate from the upstream HTTP body
|
|
1090
|
-
* stream (peer reset, abort mid-stream, DNS/transport failure after
|
|
1091
|
-
* headers were received). The handler's catch/result-mapping path looks
|
|
1092
|
-
* for this tag to return 502 `fetch_failed` instead of 500
|
|
1093
|
-
* `extraction_failed` for truncated bodies, matching the OpenAPI
|
|
1094
|
-
* contract.
|
|
1095
|
-
*/
|
|
1096
|
-
const kFetchBodyError = Symbol.for("cadet.migrationImport.fetchBodyError");
|
|
1097
|
-
|
|
1098
|
-
/**
|
|
1099
|
-
* Sidecar flag on the wrapper PassThrough indicating that its upstream
|
|
1100
|
-
* was torn down by a tagged fetch-body error. Checked after
|
|
1101
|
-
* streamCommitImport returns — the importer preserves the error message
|
|
1102
|
-
* in `result.reason = "extraction_failed"` but strips the tag.
|
|
1103
|
-
*/
|
|
1104
|
-
const kFetchBodyTornDown = Symbol.for(
|
|
1105
|
-
"cadet.migrationImport.fetchBodyTornDown",
|
|
1106
|
-
);
|
|
1107
|
-
|
|
1108
|
-
function tagFetchBodyError(err: NodeJS.ErrnoException): void {
|
|
1109
|
-
(err as unknown as Record<symbol, boolean>)[kFetchBodyError] = true;
|
|
1110
|
-
}
|
|
1111
|
-
|
|
1112
|
-
function isFetchBodyError(err: unknown): boolean {
|
|
1113
|
-
if (!err || typeof err !== "object") {
|
|
1114
|
-
return false;
|
|
1115
|
-
}
|
|
1116
|
-
return (err as unknown as Record<symbol, boolean>)[kFetchBodyError] === true;
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
function wasFetchBodyTornDown(stream: PassThrough): boolean {
|
|
1120
|
-
return (
|
|
1121
|
-
(stream as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] === true
|
|
1122
|
-
);
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
/**
|
|
1126
|
-
* Test seam shared by the import AND export URL handlers. Defaults to
|
|
1127
|
-
* `undefined`, which keeps the validator strict (GCS host, HTTPS only, no
|
|
1128
|
-
* explicit port) — the only production configuration on a normal managed
|
|
1129
|
-
* deployment. Tests override it via `_setUrlImportValidatorOptionsForTests`
|
|
1130
|
-
* (e.g. to point the validator at a local HTTP server fixture).
|
|
1131
|
-
*
|
|
1132
|
-
* NOTE: this is intentionally NOT initialized from the environment. The
|
|
1133
|
-
* `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS` env var is import-scoped and is
|
|
1134
|
-
* applied only by `resolveImportValidatorOptions` — never on the export
|
|
1135
|
-
* upload path, so a relaxed import allowlist can't widen where the
|
|
1136
|
-
* assistant is willing to PUT a bundle (SSRF). Widening the export path is
|
|
1137
|
-
* its own explicit operator opt-in via the separate
|
|
1138
|
-
* `CADET_MIGRATION_EXPORT_ALLOWED_HOSTS` env var (local/minikube only,
|
|
1139
|
-
* where the cloud serves plain-http SigV4 URLs from SeaweedFS) — see
|
|
1140
|
-
* `exportValidatorOptions`.
|
|
1141
|
-
*/
|
|
1142
|
-
let urlValidatorOptions: ValidateGcsSignedUrlOptions | undefined;
|
|
1143
|
-
|
|
1144
|
-
/**
|
|
1145
|
-
* Parse the comma-separated `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS` env value
|
|
1146
|
-
* into `ValidateGcsSignedUrlOptions`. Returns `undefined` when the value is
|
|
1147
|
-
* absent or contains no non-empty host entries, so the validator falls back
|
|
1148
|
-
* to its strict production default.
|
|
1149
|
-
*
|
|
1150
|
-
* Exported for unit testing of the env-parsing behavior.
|
|
1151
|
-
*/
|
|
1152
|
-
export function parseMigrationImportAllowedHostsEnv(
|
|
1153
|
-
raw: string | undefined,
|
|
1154
|
-
): ValidateGcsSignedUrlOptions | undefined {
|
|
1155
|
-
if (!raw) {
|
|
1156
|
-
return undefined;
|
|
1157
|
-
}
|
|
1158
|
-
const allowedHosts = raw
|
|
1159
|
-
.split(",")
|
|
1160
|
-
.map((host) => host.trim())
|
|
1161
|
-
.filter((host) => host.length > 0);
|
|
1162
|
-
return allowedHosts.length > 0 ? { allowedHosts } : undefined;
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
/**
|
|
1166
|
-
* Resolve the validator options for the IMPORT URL handlers only.
|
|
1167
|
-
*
|
|
1168
|
-
* Precedence: an explicit test override (when set) wins; otherwise the
|
|
1169
|
-
* env-derived import allowlist from `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS`.
|
|
1170
|
-
* The cloud injects that env var only where the bundle is served over
|
|
1171
|
-
* plain http from a non-GCS host — e.g. local/minikube, where the signed
|
|
1172
|
-
* bundle URL points at `http://host.docker.internal/...`. Supplying a
|
|
1173
|
-
* non-default allowlist intentionally relaxes the validator's scheme check
|
|
1174
|
-
* to also accept `http:` (and skips the explicit-port check) for those
|
|
1175
|
-
* explicitly-allowlisted hosts only; every other host still requires https.
|
|
1176
|
-
*
|
|
1177
|
-
* The export path deliberately does NOT call this — see `urlValidatorOptions`.
|
|
1178
|
-
*
|
|
1179
|
-
* Exported for unit testing.
|
|
1180
|
-
*/
|
|
1181
|
-
export function resolveImportValidatorOptions(
|
|
1182
|
-
testOverride: ValidateGcsSignedUrlOptions | undefined,
|
|
1183
|
-
rawEnv: string | undefined,
|
|
1184
|
-
): ValidateGcsSignedUrlOptions | undefined {
|
|
1185
|
-
return testOverride ?? parseMigrationImportAllowedHostsEnv(rawEnv);
|
|
1186
|
-
}
|
|
1187
|
-
|
|
1188
|
-
/**
|
|
1189
|
-
* Effective validator options for the IMPORT URL handlers, combining the
|
|
1190
|
-
* test seam with the import-only env allowlist.
|
|
1191
|
-
*/
|
|
1192
|
-
function importValidatorOptions(): ValidateGcsSignedUrlOptions | undefined {
|
|
1193
|
-
return resolveImportValidatorOptions(
|
|
1194
|
-
urlValidatorOptions,
|
|
1195
|
-
process.env.CADET_MIGRATION_IMPORT_ALLOWED_HOSTS,
|
|
1196
|
-
);
|
|
1197
|
-
}
|
|
1198
|
-
|
|
1199
|
-
/**
|
|
1200
|
-
* Effective validator options for the EXPORT (upload URL) handler,
|
|
1201
|
-
* combining the test seam with the export-only env allowlist from
|
|
1202
|
-
* `CADET_MIGRATION_EXPORT_ALLOWED_HOSTS`.
|
|
1203
|
-
*
|
|
1204
|
-
* Deliberately a separate env var from the import allowlist: the export
|
|
1205
|
-
* path controls where the assistant will PUT a full workspace bundle, so
|
|
1206
|
-
* widening it must be its own operator decision, never a side effect of
|
|
1207
|
-
* relaxing imports. The cloud injects it only where the signed upload
|
|
1208
|
-
* URL is served over plain http from a non-GCS host (local/minikube
|
|
1209
|
-
* SeaweedFS); it is unset in staging/prod, keeping the strict GCS-only
|
|
1210
|
-
* default there.
|
|
1211
|
-
*/
|
|
1212
|
-
function exportValidatorOptions(): ValidateGcsSignedUrlOptions | undefined {
|
|
1213
|
-
return resolveImportValidatorOptions(
|
|
1214
|
-
urlValidatorOptions,
|
|
1215
|
-
process.env.CADET_MIGRATION_EXPORT_ALLOWED_HOSTS,
|
|
1216
|
-
);
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
/**
|
|
1220
|
-
* Test-only: override the allowed-host list used by the URL-body import
|
|
1221
|
-
* handler. Call with `undefined` (or no arguments) to reset to production
|
|
1222
|
-
* defaults. This is intentionally not exported from the module's public
|
|
1223
|
-
* surface — tests import it directly from this file.
|
|
1224
|
-
*/
|
|
1225
|
-
export function _setUrlImportValidatorOptionsForTests(
|
|
1226
|
-
options: ValidateGcsSignedUrlOptions | undefined,
|
|
1227
|
-
): void {
|
|
1228
|
-
urlValidatorOptions = options;
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
/**
|
|
1232
|
-
* Successful outcome of `runGcsImport`. Mirrors the wire shape produced by
|
|
1233
|
-
* `importCommitSuccessResponse` (report fields spread at the top level, with
|
|
1234
|
-
* an optional `credentialsImported` summary alongside) so the same value can
|
|
1235
|
-
* be serialized directly as a Response body OR stashed as an async-job
|
|
1236
|
-
* `result` — both the sync endpoint and the async job-status endpoint then
|
|
1237
|
-
* hand the CLI a single, identical `ImportResponse`-compatible shape.
|
|
1238
|
-
*/
|
|
1239
|
-
export interface ImportSummary extends ImportCommitReport {
|
|
1240
|
-
credentialsImported?: CredentialImportSummary;
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
/**
|
|
1244
|
-
* Structured error thrown by `runGcsImport`. Carries the information needed
|
|
1245
|
-
* to reconstruct the URL-body handler's legacy Response shapes and for the
|
|
1246
|
-
* async-job registry to map to `error.code`/`upstreamStatus`.
|
|
1247
|
-
*/
|
|
1248
|
-
interface GcsImportErrorInit {
|
|
1249
|
-
code:
|
|
1250
|
-
| "invalid_url"
|
|
1251
|
-
| "fetch_failed"
|
|
1252
|
-
| "validation_failed"
|
|
1253
|
-
| "extraction_failed"
|
|
1254
|
-
| "version_incompatible"
|
|
1255
|
-
| "write_failed";
|
|
1256
|
-
message: string;
|
|
1257
|
-
upstreamStatus?: number;
|
|
1258
|
-
reason?: string;
|
|
1259
|
-
errors?: Array<{ code: string; message: string; path?: string }>;
|
|
1260
|
-
partial_report?: ImportCommitReport;
|
|
1261
|
-
/** Populated for `version_incompatible` — mirrors the cloud's PR #5470
|
|
1262
|
-
* response shape so the URL-body endpoint can return the same body. */
|
|
1263
|
-
bundle_compat?: RuntimeCompatibility;
|
|
1264
|
-
/** Populated for `version_incompatible`. */
|
|
1265
|
-
runtime_version?: string;
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
class GcsImportError extends Error {
|
|
1269
|
-
public readonly code: GcsImportErrorInit["code"];
|
|
1270
|
-
public readonly upstreamStatus?: number;
|
|
1271
|
-
public readonly reason?: string;
|
|
1272
|
-
public readonly errors?: GcsImportErrorInit["errors"];
|
|
1273
|
-
public readonly partial_report?: ImportCommitReport;
|
|
1274
|
-
public readonly bundle_compat?: RuntimeCompatibility;
|
|
1275
|
-
public readonly runtime_version?: string;
|
|
1276
|
-
|
|
1277
|
-
constructor(init: GcsImportErrorInit) {
|
|
1278
|
-
super(init.message);
|
|
1279
|
-
this.name = "GcsImportError";
|
|
1280
|
-
this.code = init.code;
|
|
1281
|
-
if (init.upstreamStatus !== undefined) {
|
|
1282
|
-
this.upstreamStatus = init.upstreamStatus;
|
|
1283
|
-
}
|
|
1284
|
-
if (init.reason !== undefined) {
|
|
1285
|
-
this.reason = init.reason;
|
|
1286
|
-
}
|
|
1287
|
-
if (init.errors !== undefined) {
|
|
1288
|
-
this.errors = init.errors;
|
|
1289
|
-
}
|
|
1290
|
-
if (init.partial_report !== undefined) {
|
|
1291
|
-
this.partial_report = init.partial_report;
|
|
1292
|
-
}
|
|
1293
|
-
if (init.bundle_compat !== undefined) {
|
|
1294
|
-
this.bundle_compat = init.bundle_compat;
|
|
1295
|
-
}
|
|
1296
|
-
if (init.runtime_version !== undefined) {
|
|
1297
|
-
this.runtime_version = init.runtime_version;
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
/**
|
|
1303
|
-
* Fetch a .vbundle from a signed GCS URL and commit it via the streaming
|
|
1304
|
-
* importer. On success, returns an `ImportSummary` the caller can serialize
|
|
1305
|
-
* into a Response or stash as an async-job `result`. On failure, throws a
|
|
1306
|
-
* `GcsImportError`:
|
|
1307
|
-
*
|
|
1308
|
-
* - `invalid_url` → URL failed `validateGcsSignedUrl` (pre-fetch).
|
|
1309
|
-
* - `fetch_failed` → upstream fetch error, non-2xx response, missing
|
|
1310
|
-
* body, OR a mid-stream body teardown tagged via
|
|
1311
|
-
* `kFetchBodyError`. `upstreamStatus` is populated
|
|
1312
|
-
* for non-2xx responses.
|
|
1313
|
-
* - `validation_failed` → the bundle failed schema/structural validation
|
|
1314
|
-
* inside `streamCommitImport`; `errors` carries
|
|
1315
|
-
* the per-issue list.
|
|
1316
|
-
* - `extraction_failed` → bundle extraction threw (malformed archive, hash
|
|
1317
|
-
* mismatch, etc.) that was NOT an upstream tear-
|
|
1318
|
-
* down. `reason` carries the importer's string.
|
|
1319
|
-
* - `write_failed` → post-extraction disk write error; `partial_report`
|
|
1320
|
-
* is attached when the importer produced one.
|
|
1321
|
-
*
|
|
1322
|
-
* The signed URL is never echoed into errors or logs — only the extracted
|
|
1323
|
-
* `host`/`path` are.
|
|
1324
|
-
*/
|
|
1325
|
-
async function runGcsImport(
|
|
1326
|
-
url: string,
|
|
1327
|
-
_correlationId?: string,
|
|
1328
|
-
): Promise<ImportSummary> {
|
|
1329
|
-
// ── 1. Validate the URL (defense-in-depth; never log the raw URL).
|
|
1330
|
-
const validated = validateGcsSignedUrl(url, importValidatorOptions());
|
|
1331
|
-
if (!validated.ok) {
|
|
1332
|
-
log.warn({ reason: validated.reason }, "Rejected migration import URL");
|
|
1333
|
-
throw new GcsImportError({
|
|
1334
|
-
code: "invalid_url",
|
|
1335
|
-
message: `Invalid URL: ${validated.reason}`,
|
|
1336
|
-
reason: validated.reason,
|
|
1337
|
-
});
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
log.info(
|
|
1341
|
-
{ host: validated.host, path: validated.path },
|
|
1342
|
-
"migration import from URL",
|
|
1343
|
-
);
|
|
1344
|
-
|
|
1345
|
-
const startedAt = Date.now();
|
|
1346
|
-
|
|
1347
|
-
// ── 2. Fetch the URL ──────────────────────────────────────────────────
|
|
1348
|
-
let upstream: Response;
|
|
1349
|
-
try {
|
|
1350
|
-
upstream = await fetch(url, {
|
|
1351
|
-
signal: AbortSignal.timeout(URL_FETCH_TIMEOUT_MS),
|
|
1352
|
-
// SSRF guard: `validateGcsSignedUrl` only vetted the initial URL.
|
|
1353
|
-
// Default fetch behavior follows 3xx responses, which would let a
|
|
1354
|
-
// validated `storage.googleapis.com` URL redirect to an arbitrary
|
|
1355
|
-
// host and bypass the allowlist. Reject redirects so we only ever
|
|
1356
|
-
// read bytes from the URL the caller handed us.
|
|
1357
|
-
redirect: "error",
|
|
1358
|
-
});
|
|
1359
|
-
} catch (err) {
|
|
1360
|
-
log.error(
|
|
1361
|
-
{
|
|
1362
|
-
host: validated.host,
|
|
1363
|
-
path: validated.path,
|
|
1364
|
-
err: err instanceof Error ? err.message : String(err),
|
|
1365
|
-
},
|
|
1366
|
-
"Failed to fetch migration import URL",
|
|
1367
|
-
);
|
|
1368
|
-
throw new GcsImportError({
|
|
1369
|
-
code: "fetch_failed",
|
|
1370
|
-
message: err instanceof Error ? err.message : String(err),
|
|
1371
|
-
});
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
if (!upstream.ok) {
|
|
1375
|
-
log.error(
|
|
1376
|
-
{
|
|
1377
|
-
host: validated.host,
|
|
1378
|
-
path: validated.path,
|
|
1379
|
-
upstream_status: upstream.status,
|
|
1380
|
-
},
|
|
1381
|
-
"Migration import URL fetch returned non-2xx",
|
|
1382
|
-
);
|
|
1383
|
-
// Drain the body so the underlying socket can be released promptly.
|
|
1384
|
-
try {
|
|
1385
|
-
await upstream.body?.cancel();
|
|
1386
|
-
} catch {
|
|
1387
|
-
/* best effort */
|
|
1388
|
-
}
|
|
1389
|
-
throw new GcsImportError({
|
|
1390
|
-
code: "fetch_failed",
|
|
1391
|
-
message: `Upstream fetch returned ${upstream.status}`,
|
|
1392
|
-
upstreamStatus: upstream.status,
|
|
1393
|
-
});
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
if (!upstream.body) {
|
|
1397
|
-
log.error(
|
|
1398
|
-
{ host: validated.host, path: validated.path },
|
|
1399
|
-
"Migration import URL fetch returned no body",
|
|
1400
|
-
);
|
|
1401
|
-
throw new GcsImportError({
|
|
1402
|
-
code: "fetch_failed",
|
|
1403
|
-
message: "Upstream fetch returned no body",
|
|
1404
|
-
});
|
|
1405
|
-
}
|
|
1406
|
-
|
|
1407
|
-
// ── 3. Stream the response through the importer ──────────────────────
|
|
1408
|
-
// Convert the WHATWG ReadableStream from fetch() into a Node Readable so
|
|
1409
|
-
// the tar-stream / gunzip / hash-verifier pipeline inside
|
|
1410
|
-
// streamCommitImport can consume it via `.pipe()`.
|
|
1411
|
-
const upstreamNodeStream = Readable.fromWeb(
|
|
1412
|
-
upstream.body as unknown as import("node:stream/web").ReadableStream<Uint8Array>,
|
|
1413
|
-
);
|
|
1414
|
-
|
|
1415
|
-
// Wrap the upstream stream in a PassThrough that tags any error bubbling
|
|
1416
|
-
// from the upstream HTTP body (peer reset, abort mid-stream, etc.) with a
|
|
1417
|
-
// known symbol. When that tagged error surfaces out of
|
|
1418
|
-
// streamCommitImport's gunzip/tar pipeline, we can distinguish it from a
|
|
1419
|
-
// legitimate bundle-format failure and map it to `fetch_failed` instead
|
|
1420
|
-
// of `extraction_failed` — matching the OpenAPI contract for the URL
|
|
1421
|
-
// body shape. We also propagate errors from the wrapper back to the
|
|
1422
|
-
// upstream stream so its underlying connection is torn down cleanly.
|
|
1423
|
-
//
|
|
1424
|
-
// Bun's `Readable.fromWeb(fetchBody)` does NOT emit `'error'` when the
|
|
1425
|
-
// TCP socket is torn down mid-response — it just emits `'close'` with
|
|
1426
|
-
// no final `'end'`. We therefore track BOTH signals:
|
|
1427
|
-
// • explicit `'error'` → tag the error, destroy the wrapper.
|
|
1428
|
-
// • premature `'close'` → synthesize an error, tag it, destroy the
|
|
1429
|
-
// wrapper. "Premature" = close fired without end first.
|
|
1430
|
-
const taggedSource = new PassThrough();
|
|
1431
|
-
let upstreamEnded = false;
|
|
1432
|
-
// True once the importer (or any local consumer) initiates a teardown of
|
|
1433
|
-
// `taggedSource`. The subsequent `close` on `upstreamNodeStream` is then a
|
|
1434
|
-
// cascaded effect of our own teardown, NOT a real upstream failure — so
|
|
1435
|
-
// we must NOT tag it as a fetch-body error, or local validation /
|
|
1436
|
-
// extraction errors would be masked as fetch_failed.
|
|
1437
|
-
let localTeardownInitiated = false;
|
|
1438
|
-
upstreamNodeStream.on("end", () => {
|
|
1439
|
-
upstreamEnded = true;
|
|
1440
|
-
});
|
|
1441
|
-
upstreamNodeStream.on("error", (err: NodeJS.ErrnoException) => {
|
|
1442
|
-
tagFetchBodyError(err);
|
|
1443
|
-
(taggedSource as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] =
|
|
1444
|
-
true;
|
|
1445
|
-
taggedSource.destroy(err);
|
|
1446
|
-
});
|
|
1447
|
-
upstreamNodeStream.on("close", () => {
|
|
1448
|
-
if (upstreamEnded) {
|
|
1449
|
-
return;
|
|
1450
|
-
}
|
|
1451
|
-
// A local teardown path closed us; don't treat this as an upstream
|
|
1452
|
-
// failure. The real error (validation / extraction / hash mismatch) is
|
|
1453
|
-
// already propagating through `streamCommitImport`'s result.
|
|
1454
|
-
if (localTeardownInitiated) {
|
|
1455
|
-
return;
|
|
1456
|
-
}
|
|
1457
|
-
const err = new Error(
|
|
1458
|
-
"Upstream body stream closed before end",
|
|
1459
|
-
) as NodeJS.ErrnoException;
|
|
1460
|
-
err.code = "ERR_UPSTREAM_BODY_CLOSED";
|
|
1461
|
-
tagFetchBodyError(err);
|
|
1462
|
-
(taggedSource as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] =
|
|
1463
|
-
true;
|
|
1464
|
-
taggedSource.destroy(err);
|
|
1465
|
-
});
|
|
1466
|
-
upstreamNodeStream.pipe(taggedSource);
|
|
1467
|
-
// Absorb stream errors on `taggedSource`. `streamCommitImport` does
|
|
1468
|
-
// several `await`s (workspace recovery, temp-dir mkdir) before
|
|
1469
|
-
// `parseVBundleStream(source)` attaches its own `source.on('error')`
|
|
1470
|
-
// listener. If the upstream socket is destroyed during that window,
|
|
1471
|
-
// the `upstreamNodeStream.on('close')` handler above calls
|
|
1472
|
-
// `taggedSource.destroy(err)` with no listener attached yet and the
|
|
1473
|
-
// error surfaces as unhandled. We don't need to act on it here — the
|
|
1474
|
-
// `kFetchBodyTornDown` flag has already been latched on the stream,
|
|
1475
|
-
// and the post-import branch below (`wasFetchBodyTornDown(taggedSource)`)
|
|
1476
|
-
// maps the failure to `fetch_failed`. Register this absorber
|
|
1477
|
-
// unconditionally so there is always at least one `'error'` listener
|
|
1478
|
-
// on the wrapper for the lifetime of the stream.
|
|
1479
|
-
taggedSource.on("error", () => {});
|
|
1480
|
-
// Propagate wrapper teardown back to the upstream fetch body. When the
|
|
1481
|
-
// streaming importer hits a validation/extraction error, it destroys
|
|
1482
|
-
// `source` (which is `taggedSource`). Without this listener the
|
|
1483
|
-
// `Readable.fromWeb(fetchBody)` stream would stay alive and continue
|
|
1484
|
-
// buffering the remote response in the background until GC or the
|
|
1485
|
-
// 60-minute timeout — a socket/bandwidth leak for any non-upstream error
|
|
1486
|
-
// (malformed bundle, hash mismatch, size cap, etc.). We set
|
|
1487
|
-
// `localTeardownInitiated` BEFORE destroying upstream so the resulting
|
|
1488
|
-
// cascaded `close` on `upstreamNodeStream` isn't misclassified as a real
|
|
1489
|
-
// upstream failure (which would return fetch_failed and mask the actual
|
|
1490
|
-
// validation error).
|
|
1491
|
-
taggedSource.on("close", () => {
|
|
1492
|
-
if (!upstreamNodeStream.destroyed) {
|
|
1493
|
-
localTeardownInitiated = true;
|
|
1494
|
-
upstreamNodeStream.destroy();
|
|
1495
|
-
}
|
|
1496
|
-
});
|
|
1497
|
-
|
|
1498
|
-
const pathResolver = new DefaultPathResolver(
|
|
1499
|
-
getWorkspaceDir(),
|
|
1500
|
-
getWorkspaceHooksDir(),
|
|
1501
|
-
);
|
|
1502
|
-
|
|
1503
|
-
// streamCommitImport does its own resetDb() internally before the atomic
|
|
1504
|
-
// swap, so we don't need to call it here.
|
|
1505
|
-
let result: ImportCommitResult;
|
|
1506
|
-
// Track credential-import outcome for inclusion in the success response.
|
|
1507
|
-
// The streaming importer invokes our callback only after the atomic swap,
|
|
1508
|
-
// so filling this in here is safe.
|
|
1509
|
-
let credentialsImported: CredentialImportSummary | undefined;
|
|
1510
|
-
// Per-invocation warning collector — scoped to this request so concurrent
|
|
1511
|
-
// URL imports can't trample each other's warnings.
|
|
1512
|
-
const credentialImportWarningSink: CredentialWarningSink = { warnings: [] };
|
|
1513
|
-
|
|
1514
|
-
try {
|
|
1515
|
-
result = await streamCommitImport({
|
|
1516
|
-
source: taggedSource,
|
|
1517
|
-
pathResolver,
|
|
1518
|
-
workspaceDir: getWorkspaceDir(),
|
|
1519
|
-
importCredentials: async (bundleCredentials) => {
|
|
1520
|
-
// We can't mutate `result.report.warnings` in place here — the
|
|
1521
|
-
// streaming importer hasn't returned its report yet. Accumulate
|
|
1522
|
-
// into a sidecar and merge into the final report below.
|
|
1523
|
-
credentialsImported = await importBundleCredentialsIntoCes(
|
|
1524
|
-
bundleCredentials,
|
|
1525
|
-
credentialImportWarningSink,
|
|
1526
|
-
);
|
|
1527
|
-
},
|
|
1528
|
-
});
|
|
1529
|
-
} catch (err) {
|
|
1530
|
-
if (isFetchBodyError(err)) {
|
|
1531
|
-
log.error(
|
|
1532
|
-
{
|
|
1533
|
-
host: validated.host,
|
|
1534
|
-
path: validated.path,
|
|
1535
|
-
err: err instanceof Error ? err.message : String(err),
|
|
1536
|
-
},
|
|
1537
|
-
"Upstream body stream failed mid-import",
|
|
1538
|
-
);
|
|
1539
|
-
throw new GcsImportError({
|
|
1540
|
-
code: "fetch_failed",
|
|
1541
|
-
message: err instanceof Error ? err.message : String(err),
|
|
1542
|
-
});
|
|
1543
|
-
}
|
|
1544
|
-
log.error(
|
|
1545
|
-
{
|
|
1546
|
-
host: validated.host,
|
|
1547
|
-
path: validated.path,
|
|
1548
|
-
err: err instanceof Error ? err.message : String(err),
|
|
1549
|
-
},
|
|
1550
|
-
"streamCommitImport threw during URL-body import",
|
|
1551
|
-
);
|
|
1552
|
-
throw new GcsImportError({
|
|
1553
|
-
code: "extraction_failed",
|
|
1554
|
-
message: err instanceof Error ? err.message : "Unexpected import error",
|
|
1555
|
-
});
|
|
1556
|
-
}
|
|
1557
|
-
|
|
1558
|
-
if (!result.ok) {
|
|
1559
|
-
// streamCommitImport swallows the raw cause and maps any
|
|
1560
|
-
// non-validation throw to `extraction_failed`. If the cause was an
|
|
1561
|
-
// upstream body failure that we tagged at the source, surface the
|
|
1562
|
-
// tag through the result (the importer preserves the message) by
|
|
1563
|
-
// detecting the latched flag on the wrapper stream.
|
|
1564
|
-
if (wasFetchBodyTornDown(taggedSource)) {
|
|
1565
|
-
log.error(
|
|
1566
|
-
{
|
|
1567
|
-
host: validated.host,
|
|
1568
|
-
path: validated.path,
|
|
1569
|
-
reason: result.reason,
|
|
1570
|
-
},
|
|
1571
|
-
"Upstream body stream failed mid-import (detected via result)",
|
|
1572
|
-
);
|
|
1573
|
-
throw new GcsImportError({
|
|
1574
|
-
code: "fetch_failed",
|
|
1575
|
-
message: "Upstream body stream failed mid-import",
|
|
1576
|
-
});
|
|
1577
|
-
}
|
|
1578
|
-
log.warn(
|
|
1579
|
-
{
|
|
1580
|
-
host: validated.host,
|
|
1581
|
-
path: validated.path,
|
|
1582
|
-
reason: result.reason,
|
|
1583
|
-
},
|
|
1584
|
-
"streamCommitImport returned failure during URL-body import",
|
|
1585
|
-
);
|
|
1586
|
-
if (result.reason === "validation_failed") {
|
|
1587
|
-
throw new GcsImportError({
|
|
1588
|
-
code: "validation_failed",
|
|
1589
|
-
message: "Bundle validation failed",
|
|
1590
|
-
reason: result.reason,
|
|
1591
|
-
errors: result.errors,
|
|
1592
|
-
});
|
|
1593
|
-
}
|
|
1594
|
-
if (result.reason === "extraction_failed") {
|
|
1595
|
-
throw new GcsImportError({
|
|
1596
|
-
code: "extraction_failed",
|
|
1597
|
-
message: result.message,
|
|
1598
|
-
reason: result.reason,
|
|
1599
|
-
});
|
|
1600
|
-
}
|
|
1601
|
-
if (result.reason === "version_incompatible") {
|
|
1602
|
-
// Returned by commitImport / streamCommitImport when the runtime falls
|
|
1603
|
-
// outside the bundle's compat range. The cloud-side gate is the
|
|
1604
|
-
// primary check; this catches legacy bundles whose ExportJob row
|
|
1605
|
-
// predates PR #5470 (compat columns NULL → platform gate skipped).
|
|
1606
|
-
throw new GcsImportError({
|
|
1607
|
-
code: "version_incompatible",
|
|
1608
|
-
message: formatRuntimeCompatibilityMessage(
|
|
1609
|
-
result.bundle_compat,
|
|
1610
|
-
result.runtime_version,
|
|
1611
|
-
),
|
|
1612
|
-
reason: result.reason,
|
|
1613
|
-
bundle_compat: result.bundle_compat,
|
|
1614
|
-
runtime_version: result.runtime_version,
|
|
1615
|
-
});
|
|
1616
|
-
}
|
|
1617
|
-
// write_failed
|
|
1618
|
-
throw new GcsImportError({
|
|
1619
|
-
code: "write_failed",
|
|
1620
|
-
message: result.message,
|
|
1621
|
-
reason: result.reason,
|
|
1622
|
-
partial_report: result.partial_report,
|
|
1623
|
-
});
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
// Merge any warnings accumulated by the credential-import callback into
|
|
1627
|
-
// the final report.
|
|
1628
|
-
if (credentialImportWarningSink.warnings.length > 0) {
|
|
1629
|
-
result.report.warnings.push(...credentialImportWarningSink.warnings);
|
|
1630
|
-
}
|
|
1631
|
-
|
|
1632
|
-
// Reconcile cadet:* metadata against CES so the gateway's
|
|
1633
|
-
// readServiceCredentials can still find platform identity values even
|
|
1634
|
-
// if Django's post-hatch provisioning raced with the streaming import
|
|
1635
|
-
// (its metadata upsert may have landed in the backup-dir copy that the
|
|
1636
|
-
// swap pushed aside, while its CES write survived on the separate
|
|
1637
|
-
// volume).
|
|
1638
|
-
await reconcileCadetMetadataFromCes(result.report);
|
|
1639
|
-
|
|
1640
|
-
// streamCommitImport already invalidated config + trust caches inside its
|
|
1641
|
-
// post-swap cleanup. We only need to check whether the newly-imported DB
|
|
1642
|
-
// carries migration checkpoints from a newer daemon version.
|
|
1643
|
-
appendNewerMigrationWarningsIfAny(result.report);
|
|
1644
|
-
|
|
1645
|
-
const elapsedMs = Date.now() - startedAt;
|
|
1646
|
-
log.info(
|
|
1647
|
-
{
|
|
1648
|
-
host: validated.host,
|
|
1649
|
-
path: validated.path,
|
|
1650
|
-
files_written: result.report.summary.files_created,
|
|
1651
|
-
bytes_written: result.report.files.reduce((n, f) => n + f.size, 0),
|
|
1652
|
-
elapsed_ms: elapsedMs,
|
|
1653
|
-
},
|
|
1654
|
-
"Migration import from URL complete",
|
|
1655
|
-
);
|
|
1656
|
-
|
|
1657
|
-
return credentialsImported
|
|
1658
|
-
? { ...result.report, credentialsImported }
|
|
1659
|
-
: { ...result.report };
|
|
1660
|
-
}
|
|
1661
|
-
|
|
1662
|
-
/**
|
|
1663
|
-
* Dispatch a JSON import body to the URL-fetch or staged-path streamer.
|
|
1664
|
-
* The body must include exactly one of `url` or `path`.
|
|
1665
|
-
*/
|
|
1666
|
-
async function handleMigrationImportFromJson(
|
|
1667
|
-
body: Record<string, unknown> | undefined,
|
|
1668
|
-
): Promise<unknown> {
|
|
1669
|
-
const hasUrl = typeof body?.url === "string";
|
|
1670
|
-
const hasPath = typeof body?.path === "string";
|
|
1671
|
-
if (hasUrl && hasPath) {
|
|
1672
|
-
throw new BadRequestError(
|
|
1673
|
-
"Request body must include exactly one of url or path",
|
|
1674
|
-
);
|
|
1675
|
-
}
|
|
1676
|
-
if (hasPath) {
|
|
1677
|
-
return handleMigrationImportFromPath(body);
|
|
1678
|
-
}
|
|
1679
|
-
return handleMigrationImportFromUrl(body);
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
async function handleMigrationImportFromPath(
|
|
1683
|
-
body: Record<string, unknown> | undefined,
|
|
1684
|
-
): Promise<unknown> {
|
|
1685
|
-
const parsed = MigrationImportPathBody.safeParse(body);
|
|
1686
|
-
if (!parsed.success) {
|
|
1687
|
-
throw new BadRequestError(
|
|
1688
|
-
"Request body must be { path: string } with a non-empty path",
|
|
1689
|
-
);
|
|
1690
|
-
}
|
|
1691
|
-
|
|
1692
|
-
let resolvedPath: string;
|
|
1693
|
-
try {
|
|
1694
|
-
resolvedPath = resolveStagedImportPath(parsed.data.path, getWorkspaceDir());
|
|
1695
|
-
} catch (err) {
|
|
1696
|
-
if (err instanceof StagedImportPathError) {
|
|
1697
|
-
throw new BadRequestError(err.message);
|
|
1698
|
-
}
|
|
1699
|
-
throw err;
|
|
1700
|
-
}
|
|
1701
|
-
|
|
1702
|
-
const source = createReadStream(resolvedPath);
|
|
1703
|
-
const pathResolver = new DefaultPathResolver(
|
|
1704
|
-
getWorkspaceDir(),
|
|
1705
|
-
getWorkspaceHooksDir(),
|
|
1706
|
-
);
|
|
1707
|
-
let credentialsImported: CredentialImportSummary | undefined;
|
|
1708
|
-
const credentialImportWarningSink: CredentialWarningSink = { warnings: [] };
|
|
1709
|
-
|
|
1710
|
-
try {
|
|
1711
|
-
const result = await streamCommitImport({
|
|
1712
|
-
source,
|
|
1713
|
-
pathResolver,
|
|
1714
|
-
workspaceDir: getWorkspaceDir(),
|
|
1715
|
-
importCredentials: async (bundleCredentials) => {
|
|
1716
|
-
credentialsImported = await importBundleCredentialsIntoCes(
|
|
1717
|
-
bundleCredentials,
|
|
1718
|
-
credentialImportWarningSink,
|
|
1719
|
-
);
|
|
1720
|
-
},
|
|
1721
|
-
});
|
|
1722
|
-
|
|
1723
|
-
if (!result.ok) {
|
|
1724
|
-
throwImportCommitFailure(result);
|
|
1725
|
-
}
|
|
1726
|
-
|
|
1727
|
-
if (credentialImportWarningSink.warnings.length > 0) {
|
|
1728
|
-
result.report.warnings.push(...credentialImportWarningSink.warnings);
|
|
1729
|
-
}
|
|
1730
|
-
|
|
1731
|
-
await reconcileCadetMetadataFromCes(result.report);
|
|
1732
|
-
appendNewerMigrationWarningsIfAny(result.report);
|
|
1733
|
-
return importCommitSuccessResult(result.report, credentialsImported);
|
|
1734
|
-
} catch (err) {
|
|
1735
|
-
if (err instanceof RouteError) {
|
|
1736
|
-
throw err;
|
|
1737
|
-
}
|
|
1738
|
-
log.error({ err }, "Unexpected error during staged-path import");
|
|
1739
|
-
throw new InternalError(
|
|
1740
|
-
err instanceof Error ? err.message : "Unexpected import error",
|
|
1741
|
-
);
|
|
1742
|
-
} finally {
|
|
1743
|
-
source.destroy();
|
|
1744
|
-
}
|
|
1745
|
-
}
|
|
1746
|
-
|
|
1747
|
-
async function handleMigrationPreflightFromPath(
|
|
1748
|
-
body: Record<string, unknown> | undefined,
|
|
1749
|
-
): Promise<unknown> {
|
|
1750
|
-
const parsed = MigrationImportPathBody.safeParse(body);
|
|
1751
|
-
if (!parsed.success) {
|
|
1752
|
-
throw new BadRequestError(
|
|
1753
|
-
"Request body must be { path: string } with a non-empty path",
|
|
1754
|
-
);
|
|
1755
|
-
}
|
|
1756
|
-
|
|
1757
|
-
let resolvedPath: string;
|
|
1758
|
-
try {
|
|
1759
|
-
resolvedPath = resolveStagedImportPath(parsed.data.path, getWorkspaceDir());
|
|
1760
|
-
} catch (err) {
|
|
1761
|
-
if (err instanceof StagedImportPathError) {
|
|
1762
|
-
throw new BadRequestError(err.message);
|
|
1763
|
-
}
|
|
1764
|
-
throw err;
|
|
1765
|
-
}
|
|
1766
|
-
|
|
1767
|
-
const source = createReadStream(resolvedPath);
|
|
1768
|
-
try {
|
|
1769
|
-
const entries = parseVBundleStream(source);
|
|
1770
|
-
const first = await entries.next();
|
|
1771
|
-
if (first.done) {
|
|
1772
|
-
return {
|
|
1773
|
-
can_import: false,
|
|
1774
|
-
validation: {
|
|
1775
|
-
is_valid: false as const,
|
|
1776
|
-
errors: [
|
|
1777
|
-
{
|
|
1778
|
-
code: "empty_archive",
|
|
1779
|
-
message: "Bundle archive is empty",
|
|
1780
|
-
},
|
|
1781
|
-
],
|
|
1782
|
-
},
|
|
1783
|
-
};
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
let manifest;
|
|
1787
|
-
try {
|
|
1788
|
-
({ manifest } = await readAndValidateManifest(first.value));
|
|
1789
|
-
} catch (err) {
|
|
1790
|
-
if (err instanceof StreamingValidationError) {
|
|
1791
|
-
return {
|
|
1792
|
-
can_import: false,
|
|
1793
|
-
validation: {
|
|
1794
|
-
is_valid: false as const,
|
|
1795
|
-
errors: [
|
|
1796
|
-
{
|
|
1797
|
-
code: err.code,
|
|
1798
|
-
message: err.message,
|
|
1799
|
-
...(err.archivePath !== undefined && { path: err.archivePath }),
|
|
1800
|
-
},
|
|
1801
|
-
],
|
|
1802
|
-
},
|
|
1803
|
-
};
|
|
1804
|
-
}
|
|
1805
|
-
throw err;
|
|
1806
|
-
}
|
|
1807
|
-
|
|
1808
|
-
for await (const entry of entries) {
|
|
1809
|
-
entry.body.resume();
|
|
1810
|
-
}
|
|
1811
|
-
|
|
1812
|
-
const pathResolver = new DefaultPathResolver(
|
|
1813
|
-
getWorkspaceDir(),
|
|
1814
|
-
getWorkspaceHooksDir(),
|
|
1815
|
-
);
|
|
1816
|
-
return analyzeImport({ manifest, pathResolver });
|
|
1817
|
-
} catch (err) {
|
|
1818
|
-
if (err instanceof RouteError) {
|
|
1819
|
-
throw err;
|
|
1820
|
-
}
|
|
1821
|
-
log.error({ err }, "Unexpected error during staged-path preflight");
|
|
1822
|
-
throw new InternalError(
|
|
1823
|
-
err instanceof Error ? err.message : "Unexpected import preflight error",
|
|
1824
|
-
);
|
|
1825
|
-
} finally {
|
|
1826
|
-
source.destroy();
|
|
1827
|
-
}
|
|
1828
|
-
}
|
|
1829
|
-
|
|
1830
|
-
/**
|
|
1831
|
-
* Handle a JSON `{ "url": "..." }` body on POST /v1/migrations/import.
|
|
1832
|
-
*
|
|
1833
|
-
* Thin wrapper around `runGcsImport` that preserves the legacy synchronous
|
|
1834
|
-
* Response shapes. `handleMigrationImportFromGcs` below uses the same helper
|
|
1835
|
-
* asynchronously via the migration-job registry.
|
|
1836
|
-
*/
|
|
1837
|
-
async function handleMigrationImportFromUrl(
|
|
1838
|
-
body: Record<string, unknown> | undefined,
|
|
1839
|
-
): Promise<unknown> {
|
|
1840
|
-
const parsed = MigrationImportUrlBody.safeParse(body);
|
|
1841
|
-
if (!parsed.success) {
|
|
1842
|
-
throw new BadRequestError(
|
|
1843
|
-
"Request body must be { url: string } with a non-empty url",
|
|
1844
|
-
);
|
|
1845
|
-
}
|
|
1846
|
-
|
|
1847
|
-
try {
|
|
1848
|
-
const summary = await runGcsImport(parsed.data.url);
|
|
1849
|
-
const { credentialsImported, ...report } = summary;
|
|
1850
|
-
return importCommitSuccessResult(report, credentialsImported);
|
|
1851
|
-
} catch (err) {
|
|
1852
|
-
throwGcsImportError(err);
|
|
1853
|
-
}
|
|
1854
|
-
}
|
|
1855
|
-
|
|
1856
|
-
/**
|
|
1857
|
-
* Map a `runGcsImport` error (or any other thrown value) to a thrown
|
|
1858
|
-
* RouteError subclass or a plain-object error body. Always throws —
|
|
1859
|
-
* callers should invoke this in a catch block.
|
|
1860
|
-
*/
|
|
1861
|
-
function throwGcsImportError(err: unknown): never {
|
|
1862
|
-
if (err instanceof GcsImportError) {
|
|
1863
|
-
if (err.code === "invalid_url") {
|
|
1864
|
-
throw new BadRequestError(err.message);
|
|
1865
|
-
}
|
|
1866
|
-
if (err.code === "fetch_failed") {
|
|
1867
|
-
throw new BadGatewayError(
|
|
1868
|
-
err.upstreamStatus
|
|
1869
|
-
? `Upstream fetch returned ${err.upstreamStatus}`
|
|
1870
|
-
: err.message,
|
|
1871
|
-
);
|
|
1872
|
-
}
|
|
1873
|
-
if (err.code === "validation_failed") {
|
|
1874
|
-
// Validation failure is not an HTTP error — return structured body
|
|
1875
|
-
// with 200 (same as raw-bytes validate path).
|
|
1876
|
-
throw new BadRequestError(
|
|
1877
|
-
JSON.stringify({
|
|
1878
|
-
success: false,
|
|
1879
|
-
reason: "validation_failed",
|
|
1880
|
-
errors: err.errors ?? [],
|
|
1881
|
-
}),
|
|
1882
|
-
);
|
|
1883
|
-
}
|
|
1884
|
-
if (err.code === "version_incompatible") {
|
|
1885
|
-
// 422 (not 500) — the bundle is structurally valid but cannot be
|
|
1886
|
-
// imported on this runtime. Body mirrors the cloud's PR #5470
|
|
1887
|
-
// response shape.
|
|
1888
|
-
throw new UnprocessableEntityError(err.message, {
|
|
1889
|
-
reason: "version_incompatible" as const,
|
|
1890
|
-
...(err.bundle_compat !== undefined && {
|
|
1891
|
-
bundle_compat: err.bundle_compat,
|
|
1892
|
-
}),
|
|
1893
|
-
...(err.runtime_version !== undefined && {
|
|
1894
|
-
runtime_version: err.runtime_version,
|
|
1895
|
-
}),
|
|
1896
|
-
});
|
|
1897
|
-
}
|
|
1898
|
-
if (err.code === "extraction_failed") {
|
|
1899
|
-
throw new InternalError(err.message);
|
|
1900
|
-
}
|
|
1901
|
-
// write_failed
|
|
1902
|
-
throw new InternalError(err.message);
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
log.error({ err }, "Unexpected error from runGcsImport");
|
|
1906
|
-
throw new InternalError(
|
|
1907
|
-
err instanceof Error ? err.message : "Unexpected import error",
|
|
1908
|
-
);
|
|
1909
|
-
}
|
|
1910
|
-
|
|
1911
|
-
/**
|
|
1912
|
-
* POST /v1/migrations/import-from-gcs
|
|
1913
|
-
*
|
|
1914
|
-
* Kick off an async bundle import from a signed GCS URL. Returns 202 with a
|
|
1915
|
-
* `job_id` the caller can poll via `GET /v1/migrations/jobs/:job_id`
|
|
1916
|
-
* (PR 4). 409 if another import is already pending or running.
|
|
1917
|
-
*
|
|
1918
|
-
* Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
|
|
1919
|
-
*/
|
|
1920
|
-
export async function handleMigrationImportFromGcs({ body }: RouteHandlerArgs) {
|
|
1921
|
-
const parsed = MigrationImportFromGcsBody.safeParse(body);
|
|
1922
|
-
if (!parsed.success) {
|
|
1923
|
-
throw new BadRequestError(
|
|
1924
|
-
"Request body must be { bundle_url: string } with a valid URL",
|
|
1925
|
-
);
|
|
1926
|
-
}
|
|
1927
|
-
|
|
1928
|
-
const { bundle_url } = parsed.data;
|
|
1929
|
-
|
|
1930
|
-
// Synchronously validate the GCS URL before consuming the single
|
|
1931
|
-
// in-flight import slot.
|
|
1932
|
-
const validated = validateGcsSignedUrl(bundle_url, importValidatorOptions());
|
|
1933
|
-
if (!validated.ok) {
|
|
1934
|
-
log.warn(
|
|
1935
|
-
{ reason: validated.reason },
|
|
1936
|
-
"Rejected migration import-from-gcs bundle URL",
|
|
1937
|
-
);
|
|
1938
|
-
throw new RouteError(
|
|
1939
|
-
`Invalid bundle URL: ${validated.reason}`,
|
|
1940
|
-
"invalid_bundle_url",
|
|
1941
|
-
400,
|
|
1942
|
-
);
|
|
1943
|
-
}
|
|
1944
|
-
|
|
1945
|
-
try {
|
|
1946
|
-
const job = migrationJobs.startJob("import", async (jobRecord) =>
|
|
1947
|
-
trackDaemonActivity({ kind: "vbundle_import" }, () =>
|
|
1948
|
-
runGcsImport(bundle_url, jobRecord.id),
|
|
1949
|
-
),
|
|
1950
|
-
);
|
|
1951
|
-
return {
|
|
1952
|
-
job_id: job.id,
|
|
1953
|
-
status: "pending" as const,
|
|
1954
|
-
type: "import" as const,
|
|
1955
|
-
};
|
|
1956
|
-
} catch (err) {
|
|
1957
|
-
if (err instanceof JobAlreadyInProgressError) {
|
|
1958
|
-
throw new RouteError(
|
|
1959
|
-
`Import already in progress: ${err.existingJobId}`,
|
|
1960
|
-
"import_in_progress",
|
|
1961
|
-
409,
|
|
1962
|
-
);
|
|
1963
|
-
}
|
|
1964
|
-
log.error({ err }, "Unexpected error scheduling import-from-gcs job");
|
|
1965
|
-
throw new InternalError(
|
|
1966
|
-
err instanceof Error ? err.message : "Unexpected import error",
|
|
1967
|
-
);
|
|
1968
|
-
}
|
|
1969
|
-
}
|
|
1970
|
-
|
|
1971
|
-
// ---------------------------------------------------------------------------
|
|
1972
|
-
// Shared helpers for raw-bytes and URL paths
|
|
1973
|
-
// ---------------------------------------------------------------------------
|
|
1974
|
-
// Preflight-from-GCS: dry-run analysis using a signed GCS download URL
|
|
1975
|
-
// ---------------------------------------------------------------------------
|
|
1976
|
-
|
|
1977
|
-
/**
|
|
1978
|
-
* POST /v1/migrations/preflight-from-gcs
|
|
1979
|
-
*
|
|
1980
|
-
* Dry-run import analysis using a signed GCS download URL. Fetches the
|
|
1981
|
-
* bundle from GCS, validates it, and returns the same preflight report as
|
|
1982
|
-
* /v1/migrations/import-preflight — without writing anything to disk.
|
|
1983
|
-
*
|
|
1984
|
-
* This allows the CLI to run `cadet teleport --dry-run` against local and
|
|
1985
|
-
* docker targets (which do not have a direct-upload preflight path like the
|
|
1986
|
-
* platform does) by reusing the same GCS-mediated bundle it would use for
|
|
1987
|
-
* a real import.
|
|
1988
|
-
*
|
|
1989
|
-
* Auth: Requires settings.write scope.
|
|
1990
|
-
*/
|
|
1991
|
-
export async function handleMigrationPreflightFromGcs({
|
|
1992
|
-
body,
|
|
1993
|
-
}: RouteHandlerArgs) {
|
|
1994
|
-
const parsed = z.object({ bundle_url: z.string().url() }).safeParse(body);
|
|
1995
|
-
if (!parsed.success) {
|
|
1996
|
-
throw new BadRequestError(
|
|
1997
|
-
"Request body must be { bundle_url: string } with a valid URL",
|
|
1998
|
-
);
|
|
1999
|
-
}
|
|
2000
|
-
|
|
2001
|
-
const { bundle_url } = parsed.data;
|
|
2002
|
-
const validated = validateGcsSignedUrl(bundle_url, importValidatorOptions());
|
|
2003
|
-
if (!validated.ok) {
|
|
2004
|
-
log.warn(
|
|
2005
|
-
{ reason: validated.reason },
|
|
2006
|
-
"Rejected preflight-from-gcs bundle URL",
|
|
2007
|
-
);
|
|
2008
|
-
throw new RouteError(
|
|
2009
|
-
`Invalid bundle URL: ${validated.reason}`,
|
|
2010
|
-
"invalid_bundle_url",
|
|
2011
|
-
400,
|
|
2012
|
-
);
|
|
2013
|
-
}
|
|
2014
|
-
|
|
2015
|
-
let upstream: Response;
|
|
2016
|
-
try {
|
|
2017
|
-
upstream = await fetch(bundle_url, {
|
|
2018
|
-
signal: AbortSignal.timeout(URL_FETCH_TIMEOUT_MS),
|
|
2019
|
-
// Reject redirects: the URL was already validated against the GCS
|
|
2020
|
-
// allowlist and we must not follow it to an arbitrary host.
|
|
2021
|
-
redirect: "error",
|
|
2022
|
-
});
|
|
2023
|
-
} catch (err) {
|
|
2024
|
-
log.error(
|
|
2025
|
-
{
|
|
2026
|
-
host: validated.host,
|
|
2027
|
-
path: validated.path,
|
|
2028
|
-
err: err instanceof Error ? err.message : String(err),
|
|
2029
|
-
},
|
|
2030
|
-
"Failed to fetch preflight-from-gcs bundle URL",
|
|
2031
|
-
);
|
|
2032
|
-
throw new InternalError(
|
|
2033
|
-
`Failed to fetch bundle: ${err instanceof Error ? err.message : String(err)}`,
|
|
2034
|
-
);
|
|
2035
|
-
}
|
|
2036
|
-
|
|
2037
|
-
if (!upstream.ok) {
|
|
2038
|
-
log.error(
|
|
2039
|
-
{
|
|
2040
|
-
host: validated.host,
|
|
2041
|
-
path: validated.path,
|
|
2042
|
-
upstream_status: upstream.status,
|
|
2043
|
-
},
|
|
2044
|
-
"preflight-from-gcs bundle URL returned non-2xx",
|
|
2045
|
-
);
|
|
2046
|
-
throw new InternalError(
|
|
2047
|
-
`Bundle URL returned ${upstream.status} — the signed URL may have expired.`,
|
|
2048
|
-
);
|
|
2049
|
-
}
|
|
2050
|
-
|
|
2051
|
-
const bytes = new Uint8Array(await upstream.arrayBuffer());
|
|
2052
|
-
const validationResult = validateVBundle(bytes, {
|
|
2053
|
-
enforceStreamingLimits: true,
|
|
2054
|
-
});
|
|
2055
|
-
|
|
2056
|
-
if (!validationResult.is_valid || !validationResult.manifest) {
|
|
2057
|
-
return {
|
|
2058
|
-
can_import: false,
|
|
2059
|
-
validation: {
|
|
2060
|
-
is_valid: false as const,
|
|
2061
|
-
errors: validationResult.errors,
|
|
2062
|
-
},
|
|
2063
|
-
};
|
|
2064
|
-
}
|
|
2065
|
-
|
|
2066
|
-
const pathResolver = new DefaultPathResolver(
|
|
2067
|
-
getWorkspaceDir(),
|
|
2068
|
-
getWorkspaceHooksDir(),
|
|
2069
|
-
);
|
|
2070
|
-
|
|
2071
|
-
try {
|
|
2072
|
-
return analyzeImport({ manifest: validationResult.manifest, pathResolver });
|
|
2073
|
-
} catch (err) {
|
|
2074
|
-
log.error({ err }, "Unexpected error during preflight-from-gcs analysis");
|
|
2075
|
-
throw new InternalError(
|
|
2076
|
-
err instanceof Error
|
|
2077
|
-
? err.message
|
|
2078
|
-
: "Unexpected preflight-from-gcs error",
|
|
2079
|
-
);
|
|
2080
|
-
}
|
|
2081
|
-
}
|
|
2082
|
-
|
|
2083
|
-
// ---------------------------------------------------------------------------
|
|
2084
|
-
|
|
2085
|
-
interface CredentialImportSummary {
|
|
2086
|
-
total: number;
|
|
2087
|
-
succeeded: number;
|
|
2088
|
-
failed: number;
|
|
2089
|
-
failedAccounts: string[];
|
|
2090
|
-
skippedCloud: number;
|
|
2091
|
-
}
|
|
2092
|
-
|
|
2093
|
-
/**
|
|
2094
|
-
* Minimal surface the credential-import helper needs to stash warnings —
|
|
2095
|
-
* either a full `ImportCommitReport` (raw-bytes path, after commitImport
|
|
2096
|
-
* returns) or an ephemeral per-request collector (streaming path, where the
|
|
2097
|
-
* report doesn't exist yet when the callback fires).
|
|
2098
|
-
*/
|
|
2099
|
-
interface CredentialWarningSink {
|
|
2100
|
-
warnings: string[];
|
|
2101
|
-
}
|
|
2102
|
-
|
|
2103
|
-
/**
|
|
2104
|
-
* Filter cloud-identity (cadet:*) credentials out of the bundle, push
|
|
2105
|
-
* user credentials into CES via `bulkSetSecureKeysAsync`, and return a
|
|
2106
|
-
* structured summary. Never throws — CES failures become report warnings.
|
|
2107
|
-
*/
|
|
2108
|
-
async function importBundleCredentialsIntoCes(
|
|
2109
|
-
bundleCredentials: Array<{ account: string; value: string }>,
|
|
2110
|
-
warningSink: CredentialWarningSink,
|
|
2111
|
-
): Promise<CredentialImportSummary | undefined> {
|
|
2112
|
-
// Filter out cloud-identity credentials (cadet:*) — these are
|
|
2113
|
-
// environment-specific and must not overwrite the target's own identity.
|
|
2114
|
-
const userCredentials = bundleCredentials.filter(
|
|
2115
|
-
(c) => !c.account.startsWith(PLATFORM_CREDENTIAL_PREFIX),
|
|
2116
|
-
);
|
|
2117
|
-
const skippedCloud = bundleCredentials.length - userCredentials.length;
|
|
2118
|
-
if (skippedCloud > 0) {
|
|
2119
|
-
log.info(`Skipped ${skippedCloud} cloud credential(s) from import`);
|
|
2120
|
-
}
|
|
2121
|
-
|
|
2122
|
-
if (userCredentials.length === 0) {
|
|
2123
|
-
if (skippedCloud > 0) {
|
|
2124
|
-
// All credentials in the bundle were cloud credentials — report
|
|
2125
|
-
// the skip count even though nothing was sent to CES.
|
|
2126
|
-
return {
|
|
2127
|
-
total: 0,
|
|
2128
|
-
succeeded: 0,
|
|
2129
|
-
failed: 0,
|
|
2130
|
-
failedAccounts: [],
|
|
2131
|
-
skippedCloud,
|
|
2132
|
-
};
|
|
2133
|
-
}
|
|
2134
|
-
return undefined;
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
try {
|
|
2138
|
-
const credResults = await bulkSetSecureKeysAsync(userCredentials);
|
|
2139
|
-
const failedResults = credResults.filter((r) => !r.ok);
|
|
2140
|
-
if (failedResults.length > 0) {
|
|
2141
|
-
log.warn(
|
|
2142
|
-
{ failed: failedResults.map((f) => f.account) },
|
|
2143
|
-
"Some credentials failed to import",
|
|
2144
|
-
);
|
|
2145
|
-
}
|
|
2146
|
-
log.info(
|
|
2147
|
-
{ total: userCredentials.length, failed: failedResults.length },
|
|
2148
|
-
"Credential import complete",
|
|
2149
|
-
);
|
|
2150
|
-
const succeeded = userCredentials.length - failedResults.length;
|
|
2151
|
-
if (failedResults.length > 0) {
|
|
2152
|
-
warningSink.warnings.push(
|
|
2153
|
-
`Imported ${succeeded} credential(s), ${failedResults.length} failed`,
|
|
2154
|
-
);
|
|
2155
|
-
}
|
|
2156
|
-
return {
|
|
2157
|
-
total: userCredentials.length,
|
|
2158
|
-
succeeded,
|
|
2159
|
-
failed: failedResults.length,
|
|
2160
|
-
failedAccounts: failedResults.map((f) => f.account),
|
|
2161
|
-
skippedCloud,
|
|
2162
|
-
};
|
|
2163
|
-
} catch (err) {
|
|
2164
|
-
log.warn({ err }, "Credential import failed entirely");
|
|
2165
|
-
warningSink.warnings.push(
|
|
2166
|
-
`Credential import failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2167
|
-
);
|
|
2168
|
-
return {
|
|
2169
|
-
total: userCredentials.length,
|
|
2170
|
-
succeeded: 0,
|
|
2171
|
-
failed: userCredentials.length,
|
|
2172
|
-
failedAccounts: userCredentials.map((c) => c.account),
|
|
2173
|
-
skippedCloud,
|
|
2174
|
-
};
|
|
2175
|
-
}
|
|
2176
|
-
}
|
|
2177
|
-
|
|
2178
|
-
/**
|
|
2179
|
-
* Append a warning to `report` when the newly-imported database contains
|
|
2180
|
-
* migration checkpoints from a daemon version newer than this one. Silent
|
|
2181
|
-
* on any validation error — the import has already succeeded.
|
|
2182
|
-
*
|
|
2183
|
-
* Gated on the report's own file counts: if the import didn't create or
|
|
2184
|
-
* overwrite any workspace files (no-swap success — e.g. credentials-only
|
|
2185
|
-
* bundle, all-skipped legacy bundle), the live DB is unchanged and any
|
|
2186
|
-
* "newer migrations" detected there came from the existing workspace,
|
|
2187
|
-
* NOT from the imported bundle. Attributing them to the bundle would be a
|
|
2188
|
-
* false positive, so skip the check entirely in that case.
|
|
2189
|
-
*/
|
|
2190
|
-
function appendNewerMigrationWarningsIfAny(report: ImportCommitReport): void {
|
|
2191
|
-
if (report.summary.files_created + report.summary.files_overwritten === 0) {
|
|
2192
|
-
return;
|
|
2193
|
-
}
|
|
2194
|
-
try {
|
|
2195
|
-
const migrationValidation = validateMigrationState(getDb(), migrationSteps);
|
|
2196
|
-
if (migrationValidation.unknownCheckpoints.length > 0) {
|
|
2197
|
-
report.warnings.push(
|
|
2198
|
-
`Imported data contains ${migrationValidation.unknownCheckpoints.length} migration(s) from a newer version. Some data may not be fully compatible.`,
|
|
2199
|
-
);
|
|
2200
|
-
}
|
|
2201
|
-
} catch {
|
|
2202
|
-
// Don't fail the import if validation itself errors
|
|
2203
|
-
}
|
|
2204
|
-
}
|
|
2205
|
-
|
|
2206
|
-
/**
|
|
2207
|
-
* Build a success result from an ImportCommitReport.
|
|
2208
|
-
*/
|
|
2209
|
-
function importCommitSuccessResult(
|
|
2210
|
-
report: ImportCommitReport,
|
|
2211
|
-
credentialsImported: CredentialImportSummary | undefined,
|
|
2212
|
-
): unknown {
|
|
2213
|
-
return {
|
|
2214
|
-
...report,
|
|
2215
|
-
...(credentialsImported ? { credentialsImported } : {}),
|
|
2216
|
-
};
|
|
2217
|
-
}
|
|
2218
|
-
|
|
2219
|
-
/**
|
|
2220
|
-
* Map an `ImportCommitResult` failure to a thrown error or a plain-object
|
|
2221
|
-
* error body. Status codes and body shapes are part of the public contract
|
|
2222
|
-
* and must remain stable.
|
|
2223
|
-
*/
|
|
2224
|
-
function throwImportCommitFailure(
|
|
2225
|
-
result: Extract<ImportCommitResult, { ok: false }>,
|
|
2226
|
-
): never {
|
|
2227
|
-
if (result.reason === "validation_failed") {
|
|
2228
|
-
// Validation failure uses 400 — structured body with error details
|
|
2229
|
-
throw new BadRequestError(
|
|
2230
|
-
JSON.stringify({
|
|
2231
|
-
success: false,
|
|
2232
|
-
reason: "validation_failed",
|
|
2233
|
-
errors: result.errors,
|
|
2234
|
-
}),
|
|
2235
|
-
);
|
|
2236
|
-
}
|
|
2237
|
-
|
|
2238
|
-
if (result.reason === "extraction_failed") {
|
|
2239
|
-
throw new InternalError(result.message);
|
|
2240
|
-
}
|
|
2241
|
-
|
|
2242
|
-
if (result.reason === "version_incompatible") {
|
|
2243
|
-
// Returned by commitImport / streamCommitImport when the runtime falls
|
|
2244
|
-
// outside the bundle's compat range. The cloud-side gate is the
|
|
2245
|
-
// primary check; this catches legacy bundles whose ExportJob row
|
|
2246
|
-
// predates PR #5470 (compat columns NULL → platform gate skipped).
|
|
2247
|
-
//
|
|
2248
|
-
// 422 (not 500) — the bundle is structurally valid but cannot be
|
|
2249
|
-
// imported on this runtime; the caller can act on it (upgrade the
|
|
2250
|
-
// runtime, choose a different bundle). Body mirrors the cloud's
|
|
2251
|
-
// PR #5470 response shape.
|
|
2252
|
-
throw new UnprocessableEntityError(
|
|
2253
|
-
formatRuntimeCompatibilityMessage(
|
|
2254
|
-
result.bundle_compat,
|
|
2255
|
-
result.runtime_version,
|
|
2256
|
-
),
|
|
2257
|
-
{
|
|
2258
|
-
reason: "version_incompatible" as const,
|
|
2259
|
-
bundle_compat: result.bundle_compat,
|
|
2260
|
-
runtime_version: result.runtime_version,
|
|
2261
|
-
},
|
|
2262
|
-
);
|
|
2263
|
-
}
|
|
2264
|
-
|
|
2265
|
-
// write_failed
|
|
2266
|
-
throw new InternalError(result.message);
|
|
2267
|
-
}
|
|
2268
|
-
|
|
2269
|
-
// ---------------------------------------------------------------------------
|
|
2270
|
-
// GET /v1/migrations/jobs/:job_id
|
|
2271
|
-
// ---------------------------------------------------------------------------
|
|
2272
|
-
|
|
2273
|
-
/**
|
|
2274
|
-
* GET /v1/migrations/jobs/:job_id
|
|
2275
|
-
*
|
|
2276
|
-
* Returns the current status of a migration job tracked by
|
|
2277
|
-
* `MigrationJobRegistry`. The response shape is a discriminated union on
|
|
2278
|
-
* `status`:
|
|
2279
|
-
*
|
|
2280
|
-
* - `{ job_id, type, status: "processing" }`
|
|
2281
|
-
* Covers both the internal `pending` and `running` states — collapsed
|
|
2282
|
-
* into a single wire value to match the cloud's transport shape used
|
|
2283
|
-
* by `ExportStatusProcessingSerializer` / `ImportStatusProcessingSerializer`.
|
|
2284
|
-
* - `{ job_id, type, status: "complete", result }`
|
|
2285
|
-
* - `{ job_id, type, status: "failed", error, error_code, upstream_status? }`
|
|
2286
|
-
*
|
|
2287
|
-
* 404 `{ error: { code: "job_not_found" } }` when no job matches the id.
|
|
2288
|
-
*/
|
|
2289
|
-
export async function handleMigrationJobStatus({
|
|
2290
|
-
pathParams,
|
|
2291
|
-
}: RouteHandlerArgs) {
|
|
2292
|
-
const jobId = pathParams?.job_id;
|
|
2293
|
-
if (!jobId) {
|
|
2294
|
-
throw new BadRequestError("Missing job_id path parameter");
|
|
2295
|
-
}
|
|
2296
|
-
|
|
2297
|
-
const job = migrationJobs.getJob(jobId);
|
|
2298
|
-
if (job === null) {
|
|
2299
|
-
throw new NotFoundError("Job not found");
|
|
2300
|
-
}
|
|
2301
|
-
|
|
2302
|
-
if (job.status === "complete") {
|
|
2303
|
-
return {
|
|
2304
|
-
job_id: job.id,
|
|
2305
|
-
type: job.type,
|
|
2306
|
-
status: "complete",
|
|
2307
|
-
result: job.result,
|
|
2308
|
-
};
|
|
2309
|
-
}
|
|
2310
|
-
|
|
2311
|
-
if (job.status === "failed") {
|
|
2312
|
-
const error = job.error;
|
|
2313
|
-
const result: Record<string, unknown> = {
|
|
2314
|
-
job_id: job.id,
|
|
2315
|
-
type: job.type,
|
|
2316
|
-
status: "failed",
|
|
2317
|
-
error: error?.message ?? "unknown",
|
|
2318
|
-
error_code: error?.code ?? "unknown",
|
|
2319
|
-
};
|
|
2320
|
-
if (error?.upstreamStatus !== undefined) {
|
|
2321
|
-
result.upstream_status = error.upstreamStatus;
|
|
2322
|
-
}
|
|
2323
|
-
return result;
|
|
2324
|
-
}
|
|
2325
|
-
|
|
2326
|
-
// pending or running — collapse to the cloud's "processing" wire value.
|
|
2327
|
-
return {
|
|
2328
|
-
job_id: job.id,
|
|
2329
|
-
type: job.type,
|
|
2330
|
-
status: "processing",
|
|
2331
|
-
};
|
|
2332
|
-
}
|
|
2333
|
-
|
|
2334
|
-
// ---------------------------------------------------------------------------
|
|
2335
|
-
// Route definitions
|
|
2336
|
-
// ---------------------------------------------------------------------------
|
|
2337
|
-
|
|
2338
|
-
export const ROUTES: RouteDefinition[] = [
|
|
2339
|
-
{
|
|
2340
|
-
operationId: "migrations_validate_post",
|
|
2341
|
-
endpoint: "migrations/validate",
|
|
2342
|
-
method: "POST",
|
|
2343
|
-
policy: {
|
|
2344
|
-
requiredScopes: ["settings.read"],
|
|
2345
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2346
|
-
},
|
|
2347
|
-
summary: "Validate a .vbundle archive",
|
|
2348
|
-
description:
|
|
2349
|
-
"Upload a .vbundle archive for validation. Accepts raw binary or multipart form data.",
|
|
2350
|
-
tags: ["migrations"],
|
|
2351
|
-
responseBody: z.object({
|
|
2352
|
-
is_valid: z.boolean(),
|
|
2353
|
-
errors: z.array(z.unknown()),
|
|
2354
|
-
manifest: z.object({}).passthrough(),
|
|
2355
|
-
}),
|
|
2356
|
-
handler: handleMigrationValidate,
|
|
2357
|
-
},
|
|
2358
|
-
{
|
|
2359
|
-
operationId: "migrations_export_post",
|
|
2360
|
-
endpoint: "migrations/export",
|
|
2361
|
-
method: "POST",
|
|
2362
|
-
policy: {
|
|
2363
|
-
requiredScopes: ["settings.write"],
|
|
2364
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2365
|
-
},
|
|
2366
|
-
summary: "Export a .vbundle archive",
|
|
2367
|
-
description:
|
|
2368
|
-
"Generate and download a .vbundle archive of the assistant's data. Optional JSON body for metadata.",
|
|
2369
|
-
tags: ["migrations"],
|
|
2370
|
-
requestBody: z.object({
|
|
2371
|
-
description: z.string().describe("Human-readable export description"),
|
|
2372
|
-
}),
|
|
2373
|
-
handler: handleMigrationExport,
|
|
2374
|
-
},
|
|
2375
|
-
{
|
|
2376
|
-
operationId: "migrations_importpreflight_post",
|
|
2377
|
-
endpoint: "migrations/import-preflight",
|
|
2378
|
-
method: "POST",
|
|
2379
|
-
policy: {
|
|
2380
|
-
requiredScopes: ["settings.write"],
|
|
2381
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2382
|
-
},
|
|
2383
|
-
summary: "Dry-run import analysis",
|
|
2384
|
-
description:
|
|
2385
|
-
"Validate a .vbundle archive and return a report of what would change on import without modifying data. Accepts raw bytes, multipart form data, or JSON `{ path }` pointing at a file staged under the workspace `.restore-staging` directory.",
|
|
2386
|
-
tags: ["migrations"],
|
|
2387
|
-
requestBody: z.object({
|
|
2388
|
-
path: z
|
|
2389
|
-
.string()
|
|
2390
|
-
.min(1)
|
|
2391
|
-
.optional()
|
|
2392
|
-
.describe(
|
|
2393
|
-
"Workspace-relative or absolute path to a staged .vbundle under .restore-staging (JSON body path only).",
|
|
2394
|
-
),
|
|
2395
|
-
}),
|
|
2396
|
-
responseBody: z.object({
|
|
2397
|
-
can_import: z.boolean(),
|
|
2398
|
-
summary: z.object({}).passthrough(),
|
|
2399
|
-
files: z.array(z.unknown()),
|
|
2400
|
-
conflicts: z.array(z.unknown()),
|
|
2401
|
-
manifest: z.object({}).passthrough(),
|
|
2402
|
-
}),
|
|
2403
|
-
handler: handleMigrationImportPreflight,
|
|
2404
|
-
},
|
|
2405
|
-
{
|
|
2406
|
-
operationId: "migrations_import_post",
|
|
2407
|
-
endpoint: "migrations/import",
|
|
2408
|
-
method: "POST",
|
|
2409
|
-
policy: {
|
|
2410
|
-
requiredScopes: ["settings.write"],
|
|
2411
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2412
|
-
},
|
|
2413
|
-
summary: "Import a .vbundle archive",
|
|
2414
|
-
description:
|
|
2415
|
-
"Commit a .vbundle archive import to disk — destructive. Accepts the bundle as raw bytes (application/octet-stream), multipart/form-data, a JSON body with `{ url }` carrying a signed URL the daemon fetches, or a JSON body with `{ path }` pointing at a file staged under the workspace `.restore-staging` directory.",
|
|
2416
|
-
tags: ["migrations"],
|
|
2417
|
-
requestBody: z.object({
|
|
2418
|
-
url: z
|
|
2419
|
-
.string()
|
|
2420
|
-
.url()
|
|
2421
|
-
.optional()
|
|
2422
|
-
.describe(
|
|
2423
|
-
"A signed GCS URL pointing to the .vbundle archive (JSON body path only).",
|
|
2424
|
-
),
|
|
2425
|
-
path: z
|
|
2426
|
-
.string()
|
|
2427
|
-
.min(1)
|
|
2428
|
-
.optional()
|
|
2429
|
-
.describe(
|
|
2430
|
-
"Workspace-relative or absolute path to a staged .vbundle under .restore-staging (JSON body path only).",
|
|
2431
|
-
),
|
|
2432
|
-
}),
|
|
2433
|
-
additionalResponses: {
|
|
2434
|
-
"502": {
|
|
2435
|
-
description: "Upstream fetch failed (URL body only).",
|
|
2436
|
-
},
|
|
2437
|
-
},
|
|
2438
|
-
responseBody: z.object({
|
|
2439
|
-
success: z.boolean(),
|
|
2440
|
-
summary: z.object({}).passthrough(),
|
|
2441
|
-
files: z.array(z.unknown()),
|
|
2442
|
-
manifest: z.object({}).passthrough(),
|
|
2443
|
-
warnings: z.array(z.unknown()),
|
|
2444
|
-
}),
|
|
2445
|
-
handler: (args) =>
|
|
2446
|
-
trackDaemonActivity({ kind: "vbundle_import" }, () =>
|
|
2447
|
-
handleMigrationImport(args),
|
|
2448
|
-
),
|
|
2449
|
-
},
|
|
2450
|
-
{
|
|
2451
|
-
operationId: "migrations_exporttogcs_post",
|
|
2452
|
-
endpoint: "migrations/export-to-gcs",
|
|
2453
|
-
method: "POST",
|
|
2454
|
-
policy: {
|
|
2455
|
-
requiredScopes: ["settings.write"],
|
|
2456
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2457
|
-
},
|
|
2458
|
-
summary: "Start an async export streamed to a GCS signed URL",
|
|
2459
|
-
description:
|
|
2460
|
-
"Kick off a background export job that PUTs a freshly-built .vbundle archive to the supplied GCS signed URL. Returns 202 with a job_id the caller can poll via the job-status endpoint. Fails fast with 409 if another export job is already pending or running.",
|
|
2461
|
-
tags: ["migrations"],
|
|
2462
|
-
requestBody: z.object({
|
|
2463
|
-
upload_url: z
|
|
2464
|
-
.string()
|
|
2465
|
-
.url()
|
|
2466
|
-
.describe("Signed GCS PUT URL that receives the exported bundle."),
|
|
2467
|
-
description: z
|
|
2468
|
-
.string()
|
|
2469
|
-
.optional()
|
|
2470
|
-
.describe("Human-readable export description."),
|
|
2471
|
-
profile: z
|
|
2472
|
-
.enum(["migration", "debug"])
|
|
2473
|
-
.optional()
|
|
2474
|
-
.describe(
|
|
2475
|
-
"Export profile. 'migration' (default) builds a teleport bundle. 'debug' builds a bundle for Cadet staff to open on a debug clone: no credentials, plus the gateway's database and logs under gateway/.",
|
|
2476
|
-
),
|
|
2477
|
-
}),
|
|
2478
|
-
responseStatus: "202",
|
|
2479
|
-
responseBody: z.object({
|
|
2480
|
-
job_id: z.string(),
|
|
2481
|
-
status: z.literal("pending"),
|
|
2482
|
-
type: z.literal("export"),
|
|
2483
|
-
}),
|
|
2484
|
-
handler: handleMigrationExportToGcs,
|
|
2485
|
-
},
|
|
2486
|
-
{
|
|
2487
|
-
operationId: "migrations_importfromgcs_post",
|
|
2488
|
-
endpoint: "migrations/import-from-gcs",
|
|
2489
|
-
method: "POST",
|
|
2490
|
-
policy: {
|
|
2491
|
-
requiredScopes: ["settings.write"],
|
|
2492
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2493
|
-
},
|
|
2494
|
-
summary: "Start an async .vbundle import from a signed GCS URL",
|
|
2495
|
-
description:
|
|
2496
|
-
"Schedule a background import job that fetches the bundle at `bundle_url` and streams it through the importer. Returns 202 with a `job_id`; poll `GET /v1/migrations/jobs/{job_id}` for status. 409 if another import is already in flight.",
|
|
2497
|
-
tags: ["migrations"],
|
|
2498
|
-
requestBody: z.object({
|
|
2499
|
-
bundle_url: z.string().url(),
|
|
2500
|
-
}),
|
|
2501
|
-
responseStatus: "202",
|
|
2502
|
-
responseBody: z.object({
|
|
2503
|
-
job_id: z.string(),
|
|
2504
|
-
status: z.literal("pending"),
|
|
2505
|
-
type: z.literal("import"),
|
|
2506
|
-
}),
|
|
2507
|
-
additionalResponses: {
|
|
2508
|
-
"409": {
|
|
2509
|
-
description: "Another import job is already pending or running.",
|
|
2510
|
-
},
|
|
2511
|
-
},
|
|
2512
|
-
handler: handleMigrationImportFromGcs,
|
|
2513
|
-
},
|
|
2514
|
-
{
|
|
2515
|
-
operationId: "migrations_preflightfromgcs_post",
|
|
2516
|
-
endpoint: "migrations/preflight-from-gcs",
|
|
2517
|
-
method: "POST",
|
|
2518
|
-
policy: {
|
|
2519
|
-
requiredScopes: ["settings.write"],
|
|
2520
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2521
|
-
},
|
|
2522
|
-
summary: "Dry-run import analysis from a signed GCS URL",
|
|
2523
|
-
description:
|
|
2524
|
-
"Fetch a .vbundle archive from a signed GCS download URL and return a preflight report — what would change if the bundle were imported — without writing anything to disk. Enables `cadet teleport --dry-run` against local and docker targets.",
|
|
2525
|
-
tags: ["migrations"],
|
|
2526
|
-
requestBody: z.object({
|
|
2527
|
-
bundle_url: z
|
|
2528
|
-
.string()
|
|
2529
|
-
.url()
|
|
2530
|
-
.describe("Signed GCS GET URL for the bundle."),
|
|
2531
|
-
}),
|
|
2532
|
-
responseBody: z.object({
|
|
2533
|
-
can_import: z.boolean(),
|
|
2534
|
-
summary: z.object({}).passthrough().optional(),
|
|
2535
|
-
files: z.array(z.unknown()).optional(),
|
|
2536
|
-
conflicts: z.array(z.unknown()).optional(),
|
|
2537
|
-
validation: z.object({}).passthrough().optional(),
|
|
2538
|
-
}),
|
|
2539
|
-
handler: handleMigrationPreflightFromGcs,
|
|
2540
|
-
},
|
|
2541
|
-
{
|
|
2542
|
-
operationId: "migrations_jobs_by_job_id_get",
|
|
2543
|
-
endpoint: "migrations/jobs/:job_id",
|
|
2544
|
-
method: "GET",
|
|
2545
|
-
policy: {
|
|
2546
|
-
requiredScopes: ["settings.read"],
|
|
2547
|
-
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
2548
|
-
},
|
|
2549
|
-
summary: "Get migration job status",
|
|
2550
|
-
description:
|
|
2551
|
-
"Return the current status of an async migration job (export or import). The response discriminates on `status`: `processing` (pending or running), `complete` (with `result`), or `failed` (with `error`, `error_code`, optional `upstream_status`).",
|
|
2552
|
-
tags: ["migrations"],
|
|
2553
|
-
pathParams: [
|
|
2554
|
-
{ name: "job_id", description: "The migration job ID to query." },
|
|
2555
|
-
],
|
|
2556
|
-
responseBody: z.discriminatedUnion("status", [
|
|
2557
|
-
z.object({
|
|
2558
|
-
job_id: z.string(),
|
|
2559
|
-
type: z.enum(["export", "import"]),
|
|
2560
|
-
status: z.literal("processing"),
|
|
2561
|
-
}),
|
|
2562
|
-
z.object({
|
|
2563
|
-
job_id: z.string(),
|
|
2564
|
-
type: z.enum(["export", "import"]),
|
|
2565
|
-
status: z.literal("complete"),
|
|
2566
|
-
result: z.unknown(),
|
|
2567
|
-
}),
|
|
2568
|
-
z.object({
|
|
2569
|
-
job_id: z.string(),
|
|
2570
|
-
type: z.enum(["export", "import"]),
|
|
2571
|
-
status: z.literal("failed"),
|
|
2572
|
-
error: z.string(),
|
|
2573
|
-
error_code: z.string(),
|
|
2574
|
-
upstream_status: z.number().int().optional(),
|
|
2575
|
-
}),
|
|
2576
|
-
]),
|
|
2577
|
-
additionalResponses: {
|
|
2578
|
-
"404": {
|
|
2579
|
-
description: "No job matches the given id.",
|
|
2580
|
-
},
|
|
2581
|
-
},
|
|
2582
|
-
handler: handleMigrationJobStatus,
|
|
2583
|
-
},
|
|
2584
|
-
];
|