@mycadet/assistant 0.12.10 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  2. package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  3. package/node_modules/@mycadet/ces-client/src/http-credentials.ts +18 -1
  4. package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  5. package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  6. package/node_modules/@mycadet/gateway-client/src/gateway-ipc-contracts.ts +0 -20
  7. package/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  8. package/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  9. package/openapi.yaml +2 -708
  10. package/package.json +2 -3
  11. package/scripts/sync-web-search-catalog.ts +1 -2
  12. package/src/__tests__/cloud-profile-guard.test.ts +41 -5
  13. package/src/__tests__/config-schema.test.ts +6 -38
  14. package/src/__tests__/conversation-queue.test.ts +45 -6
  15. package/src/__tests__/db-readiness-http-gate.test.ts +0 -6
  16. package/src/__tests__/persist-onboarding-artifacts.test.ts +0 -1
  17. package/src/__tests__/persona-resolver.test.ts +0 -51
  18. package/src/__tests__/provider-cloud-proxy-integration.test.ts +2 -110
  19. package/src/__tests__/run-due-schedules.test.ts +0 -4
  20. package/src/__tests__/schedule-store.test.ts +0 -12
  21. package/src/__tests__/stt-stream-session.test.ts +4 -4
  22. package/src/__tests__/telephony-tts-guard.test.ts +2 -50
  23. package/src/__tests__/voice-config-update.test.ts +12 -167
  24. package/src/__tests__/web-search-catalog-parity.test.ts +0 -2
  25. package/src/__tests__/workspace-migration-130-speech-mode-to-provider.test.ts +9 -7
  26. package/src/__tests__/workspace-migration-132-web-search-mode-to-provider.test.ts +6 -6
  27. package/src/calls/__tests__/voice-session-bridge.test.ts +1 -4
  28. package/src/calls/resolve-call-tts-provider.ts +6 -25
  29. package/src/calls/telephony-credential-preflight.ts +0 -10
  30. package/src/calls/telephony-tts-capability.ts +1 -17
  31. package/src/cli/commands/tts.ts +1 -2
  32. package/src/cli/index.help.ts +0 -2
  33. package/src/cli/program.ts +0 -2
  34. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +4 -42
  35. package/src/config/schemas/__tests__/stt.test.ts +10 -14
  36. package/src/config/schemas/llm.ts +3 -5
  37. package/src/config/schemas/services.ts +5 -6
  38. package/src/config/schemas/stt.ts +1 -3
  39. package/src/config/schemas/tts.ts +1 -30
  40. package/src/daemon/daemon-readiness.ts +4 -29
  41. package/src/daemon/lifecycle.ts +0 -31
  42. package/src/daemon/providers-setup.ts +0 -8
  43. package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
  44. package/src/heartbeat/heartbeat-service.ts +1 -66
  45. package/src/ipc/__tests__/db-readiness-ipc-gate.test.ts +1 -9
  46. package/src/live-voice/__tests__/live-voice-credential-preflight.test.ts +0 -44
  47. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +4 -6
  48. package/src/live-voice/__tests__/live-voice-stt.test.ts +1 -1
  49. package/src/live-voice/live-voice-credential-preflight.ts +9 -33
  50. package/src/live-voice/live-voice-session.ts +10 -6
  51. package/src/live-voice/live-voice-tts.ts +1 -8
  52. package/src/plugins/defaults/memory/worker.ts +3 -3
  53. package/src/prompts/persona-resolver.ts +0 -35
  54. package/src/providers/__tests__/search-provider-catalog.test.ts +5 -8
  55. package/src/providers/__tests__/voice-error-copy.test.ts +9 -10
  56. package/src/providers/search-provider-catalog.ts +7 -39
  57. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +3 -47
  58. package/src/providers/speech-to-text/__tests__/resolve.test.ts +6 -372
  59. package/src/providers/speech-to-text/provider-catalog.ts +2 -73
  60. package/src/providers/speech-to-text/resolve.ts +3 -120
  61. package/src/runtime/http-router-types.ts +1 -3
  62. package/src/runtime/http-server.ts +1 -1
  63. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +13 -11
  64. package/src/runtime/routes/__tests__/tts-routes.test.ts +2 -6
  65. package/src/runtime/routes/conversation-query-routes.ts +0 -96
  66. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +7 -7
  67. package/src/runtime/routes/index.ts +0 -4
  68. package/src/runtime/routes/live-voice-routes.test.ts +4 -11
  69. package/src/runtime/routes/live-voice-routes.ts +8 -11
  70. package/src/runtime/routes/secret-routes.ts +0 -6
  71. package/src/runtime/routes/tts-routes.ts +0 -38
  72. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1 -7
  73. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +0 -4
  74. package/src/schedule/schedule-store.ts +0 -7
  75. package/src/schedule/scheduler.ts +0 -5
  76. package/src/schedule/worker.ts +1 -1
  77. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +2 -10
  78. package/src/stt/daemon-batch-transcriber.ts +4 -38
  79. package/src/stt/types.ts +1 -9
  80. package/src/tools/network/__tests__/web-search.test.ts +3 -250
  81. package/src/tools/network/web-search.ts +4 -165
  82. package/src/tts/__tests__/provider-resolution.test.ts +0 -5
  83. package/src/tts/provider-catalog.ts +0 -4
  84. package/src/tts/types.ts +0 -1
  85. package/src/__tests__/cloud-speech-defaults.test.ts +0 -330
  86. package/src/__tests__/migration-cross-version-compatibility.test.ts +0 -1708
  87. package/src/__tests__/migration-export-http.test.ts +0 -557
  88. package/src/__tests__/migration-export-streaming.test.ts +0 -350
  89. package/src/__tests__/migration-export-to-gcs.test.ts +0 -706
  90. package/src/__tests__/migration-import-commit-http.test.ts +0 -1176
  91. package/src/__tests__/migration-import-from-gcs.test.ts +0 -534
  92. package/src/__tests__/migration-import-from-path.test.ts +0 -324
  93. package/src/__tests__/migration-import-from-url.test.ts +0 -588
  94. package/src/__tests__/migration-import-preflight-http.test.ts +0 -739
  95. package/src/__tests__/migration-jobs-status.test.ts +0 -139
  96. package/src/__tests__/migration-parity-persistence.test.ts +0 -1891
  97. package/src/__tests__/migration-transport.test.ts +0 -1089
  98. package/src/__tests__/migration-validate-http.test.ts +0 -634
  99. package/src/__tests__/migration-wizard.test.ts +0 -1395
  100. package/src/__tests__/rebind-secrets-screen.test.ts +0 -876
  101. package/src/__tests__/sanitize-config-for-transfer.test.ts +0 -259
  102. package/src/__tests__/transfer-progress-screen.test.ts +0 -1217
  103. package/src/__tests__/validation-results-screen.test.ts +0 -1130
  104. package/src/__tests__/vbundle-pax-and-symlink.test.ts +0 -147
  105. package/src/background-wake/background-wake-routes.test.ts +0 -906
  106. package/src/background-wake/cloud-client.test.ts +0 -328
  107. package/src/background-wake/cloud-client.ts +0 -196
  108. package/src/background-wake/next-wake.test.ts +0 -335
  109. package/src/background-wake/next-wake.ts +0 -188
  110. package/src/background-wake/publisher.ts +0 -93
  111. package/src/background-wake/wake-intent-hooks.test.ts +0 -234
  112. package/src/cli/commands/__tests__/backup.test.ts +0 -363
  113. package/src/cli/commands/backup.help.ts +0 -180
  114. package/src/cli/commands/backup.ts +0 -398
  115. package/src/cloud/cloud-speech.test.ts +0 -335
  116. package/src/cloud/cloud-speech.ts +0 -300
  117. package/src/config/cloud-speech-defaults.ts +0 -300
  118. package/src/config/sanitize-for-transfer.ts +0 -111
  119. package/src/live-voice/__tests__/live-voice-cloud-speech-fallback.test.ts +0 -250
  120. package/src/providers/speech-to-text/__tests__/cadet-cloud-realtime.test.ts +0 -457
  121. package/src/providers/speech-to-text/__tests__/cadet-cloud.test.ts +0 -271
  122. package/src/providers/speech-to-text/__tests__/cadet-speech-relay-connection.test.ts +0 -126
  123. package/src/providers/speech-to-text/cadet-cloud-flux-realtime.ts +0 -309
  124. package/src/providers/speech-to-text/cadet-cloud-realtime.ts +0 -315
  125. package/src/providers/speech-to-text/cadet-cloud.ts +0 -108
  126. package/src/providers/speech-to-text/cadet-speech-relay-connection.ts +0 -176
  127. package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +0 -254
  128. package/src/runtime/migrations/__tests__/job-registry.test.ts +0 -346
  129. package/src/runtime/migrations/__tests__/origin-mode.test.ts +0 -62
  130. package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +0 -184
  131. package/src/runtime/migrations/__tests__/staged-import-path.test.ts +0 -110
  132. package/src/runtime/migrations/__tests__/v1-test-helpers.ts +0 -112
  133. package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +0 -285
  134. package/src/runtime/migrations/__tests__/vbundle-builder-fd-leak.test.ts +0 -126
  135. package/src/runtime/migrations/__tests__/vbundle-builder-v1-shape.test.ts +0 -253
  136. package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +0 -208
  137. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +0 -485
  138. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +0 -349
  139. package/src/runtime/migrations/__tests__/vbundle-import-version-compat.test.ts +0 -196
  140. package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +0 -551
  141. package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +0 -428
  142. package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +0 -220
  143. package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +0 -1823
  144. package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +0 -598
  145. package/src/runtime/migrations/__tests__/vbundle-symlink-importer.test.ts +0 -463
  146. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -538
  147. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming.test.ts +0 -515
  148. package/src/runtime/migrations/__tests__/vbundle-symlink-tar.test.ts +0 -453
  149. package/src/runtime/migrations/__tests__/vbundle-symlink-walker.test.ts +0 -319
  150. package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +0 -240
  151. package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +0 -473
  152. package/src/runtime/migrations/gcs-signed-url.ts +0 -190
  153. package/src/runtime/migrations/job-registry.ts +0 -297
  154. package/src/runtime/migrations/migration-transport.ts +0 -612
  155. package/src/runtime/migrations/migration-wizard.ts +0 -747
  156. package/src/runtime/migrations/origin-mode.ts +0 -42
  157. package/src/runtime/migrations/rebind-secrets-screen.ts +0 -412
  158. package/src/runtime/migrations/staged-import-path.ts +0 -116
  159. package/src/runtime/migrations/transfer-progress-screen.ts +0 -323
  160. package/src/runtime/migrations/validation-results-screen.ts +0 -469
  161. package/src/runtime/migrations/vbundle-builder.ts +0 -1382
  162. package/src/runtime/migrations/vbundle-import-analyzer.ts +0 -405
  163. package/src/runtime/migrations/vbundle-import-policy.ts +0 -241
  164. package/src/runtime/migrations/vbundle-importer.ts +0 -826
  165. package/src/runtime/migrations/vbundle-metadata-merge.ts +0 -113
  166. package/src/runtime/migrations/vbundle-streaming-importer.ts +0 -2844
  167. package/src/runtime/migrations/vbundle-streaming-validator.ts +0 -426
  168. package/src/runtime/migrations/vbundle-tar-stream.ts +0 -244
  169. package/src/runtime/migrations/vbundle-validator.ts +0 -724
  170. package/src/runtime/routes/__tests__/migration-cadet-metadata-reconcile.test.ts +0 -227
  171. package/src/runtime/routes/__tests__/migration-export-secrets-redacted.test.ts +0 -54
  172. package/src/runtime/routes/__tests__/migration-import-allowed-hosts-env.test.ts +0 -70
  173. package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +0 -257
  174. package/src/runtime/routes/background-wake-routes.ts +0 -387
  175. package/src/runtime/routes/migration-routes.ts +0 -2584
  176. package/src/tools/network/__tests__/cloud-search-proxy.test.ts +0 -282
  177. package/src/tools/network/cloud-search-proxy.ts +0 -211
  178. package/src/tts/__tests__/cadet-provider.test.ts +0 -253
  179. package/src/tts/providers/cadet-provider.ts +0 -244
  180. package/src/types/tar-stream.d.ts +0 -66
@@ -1,2584 +0,0 @@
1
- /**
2
- * Route handlers for migration endpoints.
3
- *
4
- * POST /v1/migrations/validate — validate a .vbundle archive upload.
5
- * POST /v1/migrations/export — generate and download a .vbundle archive.
6
- * POST /v1/migrations/import-preflight — dry-run import analysis of a .vbundle archive.
7
- * POST /v1/migrations/import — commit a .vbundle archive import to disk.
8
- *
9
- * Accepts raw binary body (Content-Type: application/octet-stream),
10
- * multipart form data with a "file" field, or a JSON body of shape
11
- * `{ "url": "<signed-gcs-url>" }` (import only) or `{ "path": "<staged>" }`
12
- * (import and import-preflight). The path form streams a file the CLI
13
- * staged under `${workspace}/.restore-staging/`. The URL form fetches the
14
- * bundle from GCS and streams it through `streamCommitImport`.
15
- * Returns structured validation results with is_valid flag and detailed
16
- * error descriptions.
17
- */
18
-
19
- import { createReadStream } from "node:fs";
20
- import { hostname } from "node:os";
21
- import { PassThrough, Readable } from "node:stream";
22
-
23
- import { GatewayDebugExportIpcResponseSchema } from "@mycadet/gateway-client/gateway-ipc-contracts";
24
- import { z } from "zod";
25
-
26
- import { resolveCloudAssistantId } from "../../config/cloud-identity.js";
27
- import { invalidateConfigCache } from "../../config/loader.js";
28
- import { trackDaemonActivity } from "../../daemon/activity-trail.js";
29
- import { getAssistantName } from "../../daemon/identity-helpers.js";
30
- import { ipcCallPersistent } from "../../ipc/gateway-client.js";
31
- import { runAsyncSqlite } from "../../persistence/db-async-query.js";
32
- import {
33
- getDb,
34
- getLogsSqlite,
35
- getMemorySqlite,
36
- resetDb,
37
- } from "../../persistence/db-connection.js";
38
- import { validateMigrationState } from "../../persistence/migrations/validate-migration-state.js";
39
- import { migrationSteps } from "../../persistence/steps.js";
40
- import { credentialKey } from "../../security/credential-key.js";
41
- import {
42
- bulkSetSecureKeysAsync,
43
- getSecureKeyAsync,
44
- getSecureKeyResultAsync,
45
- listSecureKeysAsync,
46
- } from "../../security/secure-keys.js";
47
- import {
48
- getCredentialMetadata,
49
- upsertCredentialMetadata,
50
- } from "../../tools/credentials/metadata-store.js";
51
- import { getWorkspaceDir, getWorkspaceHooksDir } from "../../util/layout.js";
52
- import { getLogger } from "../../util/logger.js";
53
- import { APP_VERSION } from "../../version.js";
54
- import { DAEMON_INTERNAL_ASSISTANT_ID } from "../assistant-scope.js";
55
- import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
56
- import {
57
- validateGcsSignedUrl,
58
- type ValidateGcsSignedUrlOptions,
59
- } from "../migrations/gcs-signed-url.js";
60
- import {
61
- JobAlreadyInProgressError,
62
- migrationJobs,
63
- } from "../migrations/job-registry.js";
64
- import { getOriginMode } from "../migrations/origin-mode.js";
65
- import {
66
- resolveStagedImportPath,
67
- StagedImportPathError,
68
- } from "../migrations/staged-import-path.js";
69
- import type {
70
- VBundleAssistantInfo,
71
- VBundleCompatibility,
72
- VBundleExportOptions,
73
- VBundleOriginInfo,
74
- } from "../migrations/vbundle-builder.js";
75
- import { streamExportVBundle } from "../migrations/vbundle-builder.js";
76
- import {
77
- analyzeImport,
78
- DefaultPathResolver,
79
- } from "../migrations/vbundle-import-analyzer.js";
80
- import {
81
- evaluateRuntimeCompatibility,
82
- formatRuntimeCompatibilityMessage,
83
- type RuntimeCompatibility,
84
- } from "../migrations/vbundle-import-policy.js";
85
- import {
86
- commitImport,
87
- extractCredentialsFromBundle,
88
- type ImportCommitReport,
89
- type ImportCommitResult,
90
- } from "../migrations/vbundle-importer.js";
91
- import { streamCommitImport } from "../migrations/vbundle-streaming-importer.js";
92
- import {
93
- readAndValidateManifest,
94
- StreamingValidationError,
95
- } from "../migrations/vbundle-streaming-validator.js";
96
- import { parseVBundleStream } from "../migrations/vbundle-tar-stream.js";
97
- import { validateVBundle } from "../migrations/vbundle-validator.js";
98
- import {
99
- BadGatewayError,
100
- BadRequestError,
101
- InternalError,
102
- NotFoundError,
103
- RouteError,
104
- UnprocessableEntityError,
105
- } from "./errors.js";
106
- import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
107
- import { RouteResponse } from "./types.js";
108
-
109
- /**
110
- * CES account prefix for cloud-identity (`cadet:*`) credentials. Entries
111
- * with an account that starts with this string are filtered out of any
112
- * imported bundle so they don't overwrite the target's own Django-provisioned
113
- * platform identity (most notably `assistant_api_key`).
114
- *
115
- * Derived from `credentialKey("cadet", "")` so the prefix automatically
116
- * tracks the real CES account format — the literal string `"credential/cadet/"`.
117
- */
118
- const PLATFORM_CREDENTIAL_PREFIX = credentialKey("cadet", "");
119
-
120
- /**
121
- * Cadet credentials that the cloud runtime expects to see in CES.
122
- * On a cloud assistant the cloud provisions these after hatch via
123
- * `POST /v1/secrets`. A signed-in client may additionally (re)assert some of
124
- * them on teleport / local→managed transfer. Either set of writes can race with
125
- * the import — the CES write survives (separate volume), but the metadata
126
- * upsert may be clobbered by the in-place clear / atomic swap. After every
127
- * import we reconcile metadata.json against CES so any field CES already holds
128
- * a value for gets a matching metadata entry.
129
- */
130
- const CADET_CREDENTIAL_FIELDS = [
131
- "assistant_api_key",
132
- "webhook_secret",
133
- ] as const;
134
-
135
- /**
136
- * Idempotent post-import reconciliation: for each cadet:* field, if CES
137
- * has a value but metadata.json doesn't list it, upsert the entry. Pure
138
- * add-only — never deletes anything. Safe to run whether or not Django's
139
- * post-hatch provisioning has completed (missing CES values are skipped).
140
- *
141
- * Exported for direct unit-testing.
142
- */
143
- export async function reconcileCadetMetadataFromCes(warningSink: {
144
- warnings: string[];
145
- }): Promise<void> {
146
- for (const field of CADET_CREDENTIAL_FIELDS) {
147
- try {
148
- const value = await getSecureKeyAsync(credentialKey("cadet", field));
149
- if (!value) {
150
- continue;
151
- }
152
- if (getCredentialMetadata("cadet", field)) {
153
- continue;
154
- }
155
- upsertCredentialMetadata("cadet", field, {});
156
- log.info(
157
- { field },
158
- "Reconciled cadet:* metadata entry from CES after import",
159
- );
160
- } catch (err) {
161
- warningSink.warnings.push(
162
- `Failed to reconcile cadet:${field} metadata: ${
163
- err instanceof Error ? err.message : String(err)
164
- }`,
165
- );
166
- }
167
- }
168
- }
169
-
170
- const log = getLogger("migration-routes");
171
-
172
- /**
173
- * Flush each database file's WAL before an export copies them. The bundle
174
- * walker includes `*.db` but skips `*.db-wal`, so committed frames still sitting
175
- * in a WAL would be missing from the bundle.
176
- *
177
- * The main DB is flushed through `runAsyncSqlite` so the (potentially multi-GB)
178
- * flush runs in a sqlite3 subprocess where available rather than stalling the
179
- * event loop. The dedicated logs/memory connections are checkpointed in-process
180
- * on their own handles — they are small and the daemon already holds them open.
181
- * FULL (not TRUNCATE) writes committed frames back to the main file, which is
182
- * all the copy needs.
183
- */
184
- async function checkpointDbsForExport(): Promise<void> {
185
- const mainResult = await runAsyncSqlite(
186
- "PRAGMA wal_checkpoint(FULL)",
187
- "export-checkpoint:wal-checkpoint-full",
188
- );
189
- if (!mainResult.ok) {
190
- log.warn(
191
- { error: mainResult.error, backend: mainResult.backend, db: "main" },
192
- "WAL checkpoint failed — exporting without checkpoint",
193
- );
194
- }
195
-
196
- for (const [label, sqlite] of [
197
- ["logs", getLogsSqlite()],
198
- ["memory", getMemorySqlite()],
199
- // (no telemetry: migration 387 deletes that database file outright)
200
- ] as const) {
201
- if (!sqlite) {
202
- log.warn(
203
- { db: label },
204
- "Dedicated database unavailable — exporting without checkpoint",
205
- );
206
- continue;
207
- }
208
- try {
209
- sqlite.exec("PRAGMA wal_checkpoint(FULL)");
210
- } catch (err) {
211
- log.warn(
212
- { err, db: label },
213
- "WAL checkpoint failed — exporting without checkpoint",
214
- );
215
- }
216
- }
217
- }
218
-
219
- /**
220
- * Fields the export pipeline must populate on the v1 manifest.
221
- *
222
- * Centralized so both the synchronous-bytes and async-to-gcs handlers
223
- * compute the same values (and a future caller doesn't accidentally drift).
224
- */
225
- interface ExportManifestInputs {
226
- assistant: VBundleAssistantInfo;
227
- origin: VBundleOriginInfo;
228
- compatibility: VBundleCompatibility;
229
- exportOptions: VBundleExportOptions;
230
- }
231
-
232
- /**
233
- * Resolve the `assistant.id` for an export.
234
- *
235
- * In-memory identity (validate rehydration or secret-routes). The schema
236
- * requires `id` to be non-empty, so we fall back to
237
- * `DAEMON_INTERNAL_ASSISTANT_ID` rather than the empty string.
238
- */
239
- async function resolveAssistantId(): Promise<string> {
240
- const resolved = await resolveCloudAssistantId();
241
- if (resolved) {
242
- return resolved;
243
- }
244
- return DAEMON_INTERNAL_ASSISTANT_ID;
245
- }
246
-
247
- /**
248
- * Decide the truthful `secrets_redacted` flag for an export.
249
- *
250
- * The export entry points pass every collected credential through to the
251
- * builder unfiltered, so the bundle is NOT redacted whenever any
252
- * credentials made it in. Only flip to true when the credential list is
253
- * empty AND every credential read succeeded — i.e. there genuinely are
254
- * no secrets in the bundle.
255
- *
256
- * Two failure modes both force `false`:
257
- * - `storeUnreachable`: the top-level `listSecureKeysAsync()` call
258
- * failed, so we never even discovered which accounts exist.
259
- * - `perAccountUnreachable`: the LIST call succeeded but one or more
260
- * individual `getSecureKeyResultAsync(account)` reads returned
261
- * `unreachable: true`. Those accounts were silently skipped from the
262
- * `credentials` array, so a `credentialCount === 0` outcome could
263
- * reflect "we couldn't read them" rather than "no secrets exist".
264
- * Claiming a clean redaction in that case would be a lie.
265
- *
266
- * Managed-mode exports never reach this with credentials: both export
267
- * handlers skip credential collection entirely when `origin.mode` is
268
- * "managed" (the validator's cross-field refine rejects a cloud bundle
269
- * with `secrets_redacted: false`, and teleport clients re-provision
270
- * platform identity after a cloud→local import).
271
- */
272
- export function computeSecretsRedacted(
273
- credentialCount: number,
274
- storeUnreachable: boolean,
275
- perAccountUnreachable: boolean,
276
- ): boolean {
277
- return credentialCount === 0 && !storeUnreachable && !perAccountUnreachable;
278
- }
279
-
280
- /**
281
- * Compute the v1 manifest inputs that aren't tied to per-call options.
282
- *
283
- * `walkDirectoryForMetadata` skips `embedding-models`, `data/qdrant`,
284
- * `signals`, and `deprecated` — `logs` is NOT in the skip list, so log
285
- * files end up in `manifest.contents`. Browser state and memory vectors
286
- * (qdrant) are skipped, so those flags are false.
287
- */
288
- async function buildExportManifestInputs(): Promise<ExportManifestInputs> {
289
- const assistantId = await resolveAssistantId();
290
- const assistantName = getAssistantName() ?? "Assistant";
291
- const originMode = await getOriginMode();
292
- return {
293
- assistant: {
294
- id: assistantId,
295
- name: assistantName,
296
- runtime_version: APP_VERSION,
297
- },
298
- origin: {
299
- mode: originMode,
300
- hostname: hostname(),
301
- },
302
- compatibility: {
303
- min_runtime_version: APP_VERSION,
304
- max_runtime_version: null,
305
- },
306
- exportOptions: {
307
- include_logs: true,
308
- include_browser_state: false,
309
- include_memory_vectors: false,
310
- },
311
- };
312
- }
313
-
314
- /**
315
- * POST /v1/migrations/validate
316
- *
317
- * Validates a .vbundle archive. The file can be sent as:
318
- * - Raw binary body with Content-Type: application/octet-stream
319
- * - Multipart form data with a "file" field
320
- *
321
- * Returns:
322
- * 200: { is_valid: true, manifest: { ... } }
323
- * 200: { is_valid: false, errors: [{ code, message, path? }] }
324
- * 400: Standard error envelope for missing/empty body
325
- * 422: Standard error envelope for completely unparseable input
326
- */
327
- export async function handleMigrationValidate({
328
- rawBody,
329
- headers,
330
- }: RouteHandlerArgs) {
331
- const fileData = await extractFileData(rawBody, headers);
332
-
333
- try {
334
- const result = validateVBundle(fileData);
335
-
336
- return {
337
- is_valid: result.is_valid,
338
- errors: result.errors,
339
- ...(result.manifest ? { manifest: result.manifest } : {}),
340
- };
341
- } catch (err) {
342
- log.error({ err }, "Unexpected error during vbundle validation");
343
- throw new InternalError(
344
- err instanceof Error ? err.message : "Unexpected validation error",
345
- );
346
- }
347
- }
348
-
349
- /**
350
- * POST /v1/migrations/export
351
- *
352
- * Exports the assistant's real data as a .vbundle archive. The archive
353
- * contains the SQLite database (all conversations, messages, memory
354
- * segments, embeddings) and the config file.
355
- *
356
- * Accepts an optional JSON body:
357
- * { "description": "Human-readable export description" }
358
- *
359
- * Returns:
360
- * 200: Binary .vbundle archive (Content-Type: application/octet-stream)
361
- * with Content-Disposition header for download.
362
- * 500: Standard error envelope for unexpected failures.
363
- *
364
- * Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
365
- */
366
- export async function handleMigrationExport(
367
- _args: RouteHandlerArgs,
368
- ): Promise<RouteResponse> {
369
- // The legacy `description` field is no longer carried on the v1
370
- // manifest. Older clients still POST it; we silently ignore it.
371
- let cleanup: (() => Promise<void>) | undefined;
372
-
373
- try {
374
- const manifestInputs = await buildExportManifestInputs();
375
-
376
- // Managed deployments never ship credentials in bundles — the manifest
377
- // schema forbids it (`secrets_redacted` must be true when `origin.mode`
378
- // is "managed"), and teleport clients re-provision platform identity
379
- // after a cloud→local import. Redact instead of emitting a bundle
380
- // the importer is guaranteed to reject.
381
- let credentials: Array<{ account: string; value: string }> = [];
382
- let secretsRedacted = true;
383
- if (manifestInputs.origin.mode !== "managed") {
384
- // Read all stored credentials to include in the export bundle
385
- const credentialList = await listSecureKeysAsync();
386
- credentials = [];
387
- // Track per-account read failures separately from the top-level LIST
388
- // failure. A single skipped account means we cannot truthfully claim
389
- // the bundle is fully redacted — we don't know what we missed.
390
- let perAccountUnreachable = false;
391
- if (credentialList.unreachable) {
392
- log.warn(
393
- "Credential store is unreachable — export will not include credentials",
394
- );
395
- } else {
396
- for (const account of credentialList.accounts) {
397
- const result = await getSecureKeyResultAsync(account);
398
- if (result.unreachable) {
399
- perAccountUnreachable = true;
400
- log.warn(
401
- { account },
402
- "Credential store unreachable when reading credential — skipping",
403
- );
404
- } else if (result.value != null) {
405
- credentials.push({ account, value: result.value });
406
- }
407
- }
408
- }
409
- secretsRedacted = computeSecretsRedacted(
410
- credentials.length,
411
- credentialList.unreachable,
412
- perAccountUnreachable,
413
- );
414
- }
415
-
416
- const result = await streamExportVBundle({
417
- workspaceDir: getWorkspaceDir(),
418
- ...manifestInputs,
419
- secretsRedacted,
420
- credentials,
421
- checkpoint: checkpointDbsForExport,
422
- });
423
-
424
- cleanup = result.cleanup;
425
- const { tempPath, size, manifest } = result;
426
-
427
- const timestamp = manifest.created_at.replace(/[:.]/g, "-");
428
- const filename = `export-${timestamp}.vbundle`;
429
-
430
- const fileStream = createReadStream(tempPath);
431
- fileStream.on("close", () => {
432
- cleanup?.();
433
- cleanup = undefined;
434
- });
435
-
436
- const streamBody = Readable.toWeb(fileStream) as unknown as ReadableStream;
437
-
438
- return new RouteResponse(streamBody, {
439
- "Content-Type": "application/octet-stream",
440
- "Content-Disposition": `attachment; filename="${filename}"`,
441
- "Content-Length": String(size),
442
- // `schema_version` is now an integer; clients that parse this header
443
- // continue to work, but the value flips from "1.0" to "1".
444
- "X-Vbundle-Schema-Version": String(manifest.schema_version),
445
- // Header name preserved for cross-version client compat; populated
446
- // from the renamed manifest `checksum` field.
447
- "X-Vbundle-Manifest-Sha256": manifest.checksum,
448
- "X-Vbundle-Credentials-Included": String(credentials.length),
449
- });
450
- } catch (err) {
451
- await cleanup?.();
452
- log.error({ err }, "Failed to build export bundle");
453
- throw new InternalError(
454
- err instanceof Error ? err.message : "Unexpected export error",
455
- );
456
- }
457
- }
458
-
459
- // ---------------------------------------------------------------------------
460
- // POST /v1/migrations/export-to-gcs — async export streamed to a signed URL
461
- // ---------------------------------------------------------------------------
462
-
463
- /** 60 minutes — matches the URL-body import fetch deadline. */
464
- const EXPORT_TO_GCS_PUT_TIMEOUT_MS = 60 * 60 * 1000;
465
-
466
- const MigrationExportToGcsBody = z.object({
467
- upload_url: z.string().url(),
468
- description: z.string().optional(),
469
- /**
470
- * `debug`: a bundle for Cadet staff to inspect. Adds the gateway's
471
- * database and recent logs under `gateway/`. `migration` is the default.
472
- */
473
- profile: z.enum(["migration", "debug"]).default("migration"),
474
- });
475
-
476
- /** Archive path of the gateway's contribution to a debug bundle. */
477
- export const GATEWAY_DEBUG_EXPORT_PATH = "gateway/export.tar.gz";
478
-
479
- /**
480
- * Ask the gateway for its debug archive over the local socket. The daemon
481
- * cannot read the gateway's files itself: in local mode they sit in the
482
- * protected directory beside the encryption keys, and in Docker on a volume
483
- * the daemon does not mount.
484
- */
485
- async function collectGatewayDebugExport(): Promise<{
486
- archivePath: string;
487
- data: Uint8Array;
488
- }> {
489
- const raw = await ipcCallPersistent(
490
- "gateway_debug_export",
491
- {},
492
- GATEWAY_DEBUG_EXPORT_TIMEOUT_MS,
493
- );
494
- const parsed = GatewayDebugExportIpcResponseSchema.parse(raw);
495
- return {
496
- archivePath: GATEWAY_DEBUG_EXPORT_PATH,
497
- data: new Uint8Array(Buffer.from(parsed.archive_base64, "base64")),
498
- };
499
- }
500
-
501
- const GATEWAY_DEBUG_EXPORT_TIMEOUT_MS = 120_000;
502
-
503
- /**
504
- * Collected credentials plus warning markers if the credential store was
505
- * unreachable. The caller surfaces the warning in logs; production callers
506
- * fail closed on errors (a thrown exception → 500) to avoid shipping a
507
- * bundle with partial credentials. An unreachable store is NOT an error —
508
- * `handleMigrationExport` treats that case as "export without credentials".
509
- *
510
- * - `unreachable`: the top-level `listSecureKeysAsync()` call failed.
511
- * - `perAccountUnreachable`: the LIST succeeded but one or more individual
512
- * `getSecureKeyResultAsync(account)` calls returned `unreachable: true`.
513
- * Those accounts were silently skipped from `credentials`, so the count
514
- * here understates reality. The flag is what tells `computeSecretsRedacted`
515
- * it cannot claim a clean redaction.
516
- */
517
- interface CollectedCredentials {
518
- credentials: Array<{ account: string; value: string }>;
519
- unreachable: boolean;
520
- perAccountUnreachable: boolean;
521
- }
522
-
523
- /**
524
- * Mirror of the credential-collection block inside `handleMigrationExport`.
525
- * Factored out so the new async export-to-gcs handler can share the exact
526
- * same behavior. Throws if the credential store raises an unexpected error —
527
- * the caller translates that into a 500 (fail closed on credential errors).
528
- */
529
- async function collectExportCredentials(): Promise<CollectedCredentials> {
530
- const credentialList = await listSecureKeysAsync();
531
- if (credentialList.unreachable) {
532
- log.warn(
533
- "Credential store is unreachable — export will not include credentials",
534
- );
535
- return {
536
- credentials: [],
537
- unreachable: true,
538
- perAccountUnreachable: false,
539
- };
540
- }
541
- const credentials: Array<{ account: string; value: string }> = [];
542
- let perAccountUnreachable = false;
543
- for (const account of credentialList.accounts) {
544
- const result = await getSecureKeyResultAsync(account);
545
- if (result.unreachable) {
546
- perAccountUnreachable = true;
547
- log.warn(
548
- { account },
549
- "Credential store unreachable when reading credential — skipping",
550
- );
551
- } else if (result.value != null) {
552
- credentials.push({ account, value: result.value });
553
- }
554
- }
555
- return { credentials, unreachable: false, perAccountUnreachable };
556
- }
557
-
558
- /**
559
- * POST /v1/migrations/export-to-gcs
560
- *
561
- * Starts an async export job that streams a freshly-built .vbundle archive
562
- * to a GCS signed PUT URL. Returns `202 Accepted` with a `job_id` the caller
563
- * can poll via the job-status endpoint; the bundle upload runs in the
564
- * background via `migrationJobs`.
565
- *
566
- * Request body (JSON):
567
- * { upload_url: string, description?: string }
568
- *
569
- * Responses:
570
- * 202: { job_id, status: "pending", type: "export" }
571
- * 400: { error: { code: "invalid_upload_url", reason } } — URL failed
572
- * `validateGcsSignedUrl` (scheme/host/signature/traversal).
573
- * 409: { error: { code: "export_in_progress", job_id } } — another export
574
- * job is already pending or running.
575
- * 500: Standard error envelope for credential-collection failures or
576
- * other unexpected errors before the job is enqueued.
577
- *
578
- * Terminal job state (surfaced via the job-status endpoint once poll lands):
579
- * result: { size, sha256, schemaVersion, credentialsIncluded }
580
- * error.code = "upload_failed" with `upstreamStatus` on non-2xx PUT
581
- * error.code = "fetch_failed" on transport errors from the PUT itself.
582
- *
583
- * Auth: settings.write scope (matches `migrations/export`).
584
- */
585
- export async function handleMigrationExportToGcs({ body }: RouteHandlerArgs) {
586
- // ── 1. Parse JSON body ────────────────────────────────────────────────
587
- const parsed = MigrationExportToGcsBody.safeParse(body);
588
- if (!parsed.success) {
589
- throw new BadRequestError(
590
- "Request body must be { upload_url: string, description?: string, profile?: 'migration' | 'debug' } with a valid URL",
591
- );
592
- }
593
- const isDebugProfile = parsed.data.profile === "debug";
594
-
595
- // ── 2. Validate the upload URL. Never log `parsed.data.upload_url`.
596
- const validated = validateGcsSignedUrl(
597
- parsed.data.upload_url,
598
- exportValidatorOptions(),
599
- );
600
- if (!validated.ok) {
601
- log.warn(
602
- { reason: validated.reason },
603
- "Rejected migration export-to-gcs upload URL",
604
- );
605
- throw new RouteError(
606
- `Invalid upload URL: ${validated.reason}`,
607
- "invalid_upload_url",
608
- 400,
609
- );
610
- }
611
-
612
- log.info(
613
- { host: validated.host, path: validated.path },
614
- "migration export to GCS starting",
615
- );
616
-
617
- const uploadUrl = parsed.data.upload_url;
618
-
619
- // Compute the v1 manifest inputs once outside the async job runner so we
620
- // surface failures (e.g. credential-store probe) as a synchronous 500
621
- // before the caller starts polling.
622
- let manifestInputs: ExportManifestInputs;
623
- try {
624
- manifestInputs = await buildExportManifestInputs();
625
- } catch (err) {
626
- log.error({ err }, "Failed to assemble export manifest inputs");
627
- throw new InternalError(
628
- err instanceof Error
629
- ? err.message
630
- : "Failed to assemble export manifest inputs",
631
- );
632
- }
633
-
634
- // ── 3. Collect credentials up front. Fail closed → 500. Managed
635
- // deployments never ship credentials in bundles — the manifest schema
636
- // forbids it (`secrets_redacted` must be true when `origin.mode` is
637
- // "managed"), and teleport clients re-provision platform identity after
638
- // a cloud→local import — so redact instead of emitting a bundle the
639
- // importer is guaranteed to reject.
640
- //
641
- // A debug bundle goes to Cadet staff, so it never carries credentials
642
- // either, whatever the deployment mode.
643
- let collected: CollectedCredentials;
644
- if (manifestInputs.origin.mode === "managed" || isDebugProfile) {
645
- collected = {
646
- credentials: [],
647
- unreachable: false,
648
- perAccountUnreachable: false,
649
- };
650
- } else {
651
- try {
652
- collected = await collectExportCredentials();
653
- } catch (err) {
654
- log.error({ err }, "Failed to collect credentials for export-to-gcs");
655
- throw new InternalError(
656
- err instanceof Error ? err.message : "Failed to collect credentials",
657
- );
658
- }
659
- }
660
-
661
- const secretsRedacted = computeSecretsRedacted(
662
- collected.credentials.length,
663
- collected.unreachable,
664
- collected.perAccountUnreachable,
665
- );
666
-
667
- if (isDebugProfile) {
668
- manifestInputs = {
669
- ...manifestInputs,
670
- exportOptions: { ...manifestInputs.exportOptions, include_gateway: true },
671
- };
672
- }
673
-
674
- // ── 4. Enqueue the job. The runner captures the collected credentials.
675
- let job;
676
- try {
677
- job = migrationJobs.startJob("export", async () => {
678
- let cleanup: (() => Promise<void>) | undefined;
679
- try {
680
- // The gateway snapshot can take a while on a large database, so it
681
- // runs inside the job (after the export slot is held) rather than
682
- // before the 202. A debug bundle without the gateway's data is not
683
- // what staff asked for, so a failure fails the job and nothing is
684
- // uploaded.
685
- let extraFiles: Array<{ archivePath: string; data: Uint8Array }> = [];
686
- if (isDebugProfile) {
687
- try {
688
- extraFiles = [await collectGatewayDebugExport()];
689
- } catch (err) {
690
- log.error({ err }, "Failed to collect the gateway's debug export");
691
- const wrapped = new Error(
692
- "The gateway did not provide its debug export; the bundle was not sent.",
693
- );
694
- (wrapped as { code?: string }).code = "gateway_debug_export_failed";
695
- throw wrapped;
696
- }
697
- }
698
-
699
- const result = await streamExportVBundle({
700
- workspaceDir: getWorkspaceDir(),
701
- ...manifestInputs,
702
- secretsRedacted,
703
- credentials: collected.credentials,
704
- extraFiles,
705
- checkpoint: checkpointDbsForExport,
706
- });
707
-
708
- cleanup = result.cleanup;
709
- const { tempPath, size, manifest } = result;
710
-
711
- // Stream the temp file to GCS via PUT. Using Node's ReadableStream
712
- // bridge keeps peak memory bounded — we do NOT load the archive
713
- // into memory.
714
- const fileStream = createReadStream(tempPath);
715
- const webBody = Readable.toWeb(
716
- fileStream,
717
- ) as unknown as ReadableStream<Uint8Array>;
718
-
719
- let response: Response;
720
- try {
721
- response = await fetch(uploadUrl, {
722
- method: "PUT",
723
- body: webBody,
724
- // `duplex: "half"` is required when sending a streaming body
725
- // via fetch in Node/Bun — without it the cloud rejects the
726
- // request as "duplex option is required when body is a
727
- // ReadableStream".
728
- duplex: "half",
729
- // `validateGcsSignedUrl` only vets the initial URL. If the
730
- // upstream responds with a 3xx, default fetch would follow
731
- // the redirect and PUT bytes to an attacker-controlled host.
732
- // Refuse redirects so the signed URL's origin is the only
733
- // destination for the export archive.
734
- redirect: "error",
735
- headers: {
736
- "Content-Type": "application/octet-stream",
737
- "Content-Length": String(size),
738
- },
739
- signal: AbortSignal.timeout(EXPORT_TO_GCS_PUT_TIMEOUT_MS),
740
- } as RequestInit & { duplex: "half" });
741
- } catch (err) {
742
- // Transport-level fetch failures (DNS, reset, abort) — tag them
743
- // with the fetch-body marker so the registry maps them to
744
- // `error.code = "fetch_failed"` and logs stay consistent with
745
- // the import URL path.
746
- const wrapped = err instanceof Error ? err : new Error(String(err));
747
- tagFetchBodyError(wrapped as NodeJS.ErrnoException);
748
- (wrapped as { code?: string }).code = "fetch_failed";
749
- throw wrapped;
750
- }
751
-
752
- if (!response.ok) {
753
- // Drain so the socket is released promptly. Ignore drain errors.
754
- try {
755
- await response.body?.cancel();
756
- } catch {
757
- /* best-effort */
758
- }
759
- const uploadErr = new Error(
760
- `Upload to GCS failed with status ${response.status}`,
761
- );
762
- (uploadErr as { code?: string }).code = "upload_failed";
763
- (uploadErr as { upstreamStatus?: number }).upstreamStatus =
764
- response.status;
765
- throw uploadErr;
766
- }
767
-
768
- return {
769
- size,
770
- sha256: manifest.checksum,
771
- schemaVersion: manifest.schema_version,
772
- credentialsIncluded: collected.credentials.length,
773
- };
774
- } finally {
775
- // Mirror the raw-bytes export cleanup pattern: the stream's
776
- // `close` listener is the happy-path cleanup in that handler, but
777
- // here we keep everything in the async block, so a finally is the
778
- // right place to evict the temp file regardless of outcome.
779
- if (cleanup) {
780
- try {
781
- await cleanup();
782
- } catch (err) {
783
- log.warn({ err }, "Failed to clean up export-to-gcs temp file");
784
- }
785
- }
786
- }
787
- });
788
- } catch (err) {
789
- if (err instanceof JobAlreadyInProgressError) {
790
- throw new RouteError(
791
- `Export already in progress: ${err.existingJobId}`,
792
- "export_in_progress",
793
- 409,
794
- );
795
- }
796
- log.error({ err }, "Unexpected error while enqueueing export-to-gcs job");
797
- throw new InternalError(
798
- err instanceof Error ? err.message : "Unexpected export-to-gcs error",
799
- );
800
- }
801
-
802
- return {
803
- job_id: job.id,
804
- status: "pending" as const,
805
- type: "export" as const,
806
- };
807
- }
808
-
809
- /**
810
- * Extract file data from a request body, supporting both raw binary
811
- * and multipart form data uploads.
812
- *
813
- * Shared between validate and import-preflight handlers.
814
- */
815
- async function extractFileData(
816
- rawBody: Uint8Array | undefined,
817
- headers: Record<string, string> | undefined,
818
- ): Promise<Uint8Array> {
819
- const contentType = headers?.["content-type"] ?? "";
820
-
821
- if (contentType.includes("multipart/form-data")) {
822
- if (!rawBody) {
823
- throw new BadRequestError("Request body is empty");
824
- }
825
- try {
826
- const syntheticReq = new Request("http://localhost", {
827
- method: "POST",
828
- headers: { "content-type": contentType },
829
- body: rawBody.buffer as ArrayBuffer,
830
- });
831
- const formData = await syntheticReq.formData();
832
- const file = formData.get("file");
833
- if (!file || !(file instanceof Blob)) {
834
- throw new BadRequestError('Multipart upload requires a "file" field');
835
- }
836
- return new Uint8Array(await file.arrayBuffer());
837
- } catch (err) {
838
- if (err instanceof BadRequestError) {
839
- throw err;
840
- }
841
- log.error({ err }, "Failed to parse multipart form data");
842
- throw new BadRequestError("Invalid multipart form data");
843
- }
844
- }
845
-
846
- // Raw binary body — already provided as rawBody by the adapter
847
- if (!rawBody || rawBody.length === 0) {
848
- throw new BadRequestError(
849
- "Request body is empty — a .vbundle file is required",
850
- );
851
- }
852
- return rawBody;
853
- }
854
-
855
- /**
856
- * POST /v1/migrations/import-preflight
857
- *
858
- * Dry-run import analysis. Accepts a .vbundle archive upload, validates it,
859
- * and returns a detailed report of what would change if the bundle were
860
- * actually imported — without modifying any data on disk.
861
- *
862
- * The file can be sent as:
863
- * - Raw binary body with Content-Type: application/octet-stream
864
- * - Multipart form data with a "file" field
865
- * - JSON body `{ "path": "<staged-relative-or-absolute>" }` pointing at a
866
- * `.vbundle` the caller placed under `${workspace}/.restore-staging/`
867
- *
868
- * Returns:
869
- * 200: {
870
- * can_import: boolean,
871
- * summary: { total_files, files_to_create, files_to_overwrite, files_unchanged },
872
- * files: [{ path, action, bundle_size, current_size, bundle_sha256, current_sha256 }],
873
- * conflicts: [{ code, message, path? }],
874
- * manifest: { ... }
875
- * }
876
- * 200: { can_import: false, validation: { is_valid: false, errors: [...] } }
877
- * (when the bundle itself is invalid)
878
- * 400: Standard error envelope for missing/empty body
879
- * 500: Standard error envelope for unexpected failures
880
- *
881
- * Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
882
- */
883
- export async function handleMigrationImportPreflight({
884
- rawBody,
885
- headers,
886
- body,
887
- }: RouteHandlerArgs) {
888
- const contentType = headers?.["content-type"] ?? "";
889
- if (contentType.includes("application/json")) {
890
- return handleMigrationPreflightFromPath(body);
891
- }
892
-
893
- const fileData = await extractFileData(rawBody, headers);
894
-
895
- try {
896
- const validationResult = validateVBundle(fileData);
897
-
898
- if (!validationResult.is_valid || !validationResult.manifest) {
899
- return {
900
- can_import: false,
901
- validation: {
902
- is_valid: false,
903
- errors: validationResult.errors,
904
- },
905
- };
906
- }
907
-
908
- const pathResolver = new DefaultPathResolver(
909
- getWorkspaceDir(),
910
- getWorkspaceHooksDir(),
911
- );
912
-
913
- return analyzeImport({
914
- manifest: validationResult.manifest,
915
- pathResolver,
916
- });
917
- } catch (err) {
918
- log.error({ err }, "Unexpected error during import preflight analysis");
919
- throw new InternalError(
920
- err instanceof Error ? err.message : "Unexpected import preflight error",
921
- );
922
- }
923
- }
924
-
925
- /**
926
- * POST /v1/migrations/import
927
- *
928
- * Commits a .vbundle archive import to disk. This is a destructive operation
929
- * that writes bundle files to their target locations, replacing existing data.
930
- *
931
- * The import process:
932
- * 1. Validates the bundle (validation before any state mutation)
933
- * 2. Extracts files from the archive
934
- * 3. Backs up existing files before overwriting
935
- * 4. Writes bundle files to disk
936
- * 5. Verifies post-write integrity (SHA-256 check)
937
- * 6. Returns a detailed report of what was imported
938
- *
939
- * The bundle can be supplied as:
940
- * - Raw binary body with Content-Type: application/octet-stream
941
- * - Multipart form data with a "file" field
942
- * - JSON body `{ "url": "<signed-gcs-url>" }` (Content-Type:
943
- * application/json). The daemon fetches and streams the archive
944
- * through `streamCommitImport`, so peak memory stays bounded by a
945
- * single tar entry rather than bundle size.
946
- * - JSON body `{ "path": "<staged>" }` pointing at a `.vbundle` the
947
- * caller placed under `${workspace}/.restore-staging/`. The daemon
948
- * streams that file through `streamCommitImport`.
949
- *
950
- * Returns:
951
- * 200: {
952
- * success: true,
953
- * summary: { total_files, files_created, files_overwritten, files_skipped, backups_created },
954
- * files: [{ path, disk_path, action, size, sha256, backup_path }],
955
- * manifest: { ... },
956
- * warnings: [...]
957
- * }
958
- * 200: { success: false, reason: "validation_failed", errors: [...] }
959
- * 400: Standard error envelope for missing/empty body or malformed URL
960
- * 500: Standard error envelope for unexpected failures
961
- * 502: { success: false, reason: "fetch_failed", upstream_status?: number }
962
- * (URL path only — upstream GCS fetch failed)
963
- *
964
- * Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
965
- */
966
- export async function handleMigrationImport(
967
- args: RouteHandlerArgs,
968
- ): Promise<unknown> {
969
- const { body, rawBody, headers } = args;
970
- // JSON body is either a signed URL fetch or a staged local-file stream.
971
- const contentType = headers?.["content-type"] ?? "";
972
- if (contentType.includes("application/json")) {
973
- return handleMigrationImportFromJson(body);
974
- }
975
-
976
- const fileData = await extractFileData(rawBody, headers);
977
-
978
- try {
979
- // Validate the bundle before closing the DB to avoid an unnecessary
980
- // close/reopen cycle when the bundle is invalid. Pass the validated
981
- // manifest and entries to commitImport so it skips re-validation
982
- // (avoids holding two copies of decompressed data in memory).
983
- const validation = validateVBundle(fileData);
984
- if (!validation.is_valid) {
985
- return {
986
- success: false,
987
- reason: "validation_failed",
988
- errors: validation.errors,
989
- };
990
- }
991
-
992
- // Pre-check runtime-version compat before the DB close/reopen cycle.
993
- // commitImport runs the same gate as defense-in-depth for callers that
994
- // don't pre-check; we run it here too so an incompatible bundle short-
995
- // circuits before resetDb().
996
- const compatResult = evaluateRuntimeCompatibility(
997
- validation.manifest!.compatibility,
998
- APP_VERSION,
999
- );
1000
- if (!compatResult.ok) {
1001
- throwImportCommitFailure({
1002
- ok: false,
1003
- reason: "version_incompatible",
1004
- bundle_compat: compatResult.bundle_compat,
1005
- runtime_version: compatResult.runtime_version,
1006
- });
1007
- }
1008
-
1009
- const pathResolver = new DefaultPathResolver(
1010
- getWorkspaceDir(),
1011
- getWorkspaceHooksDir(),
1012
- );
1013
-
1014
- // Close the live SQLite connection before overwriting assistant.db on disk.
1015
- // The singleton will be lazily reopened on the next getDb() call.
1016
- resetDb();
1017
-
1018
- const result = commitImport({
1019
- archiveData: fileData,
1020
- pathResolver,
1021
- preValidatedManifest: validation.manifest,
1022
- preValidatedEntries: validation.entries,
1023
- workspaceDir: getWorkspaceDir(),
1024
- });
1025
-
1026
- if (!result.ok) {
1027
- throwImportCommitFailure(result);
1028
- }
1029
-
1030
- // Import credentials from the bundle into CES (non-blocking — failures
1031
- // are logged as warnings but do not fail the overall import).
1032
- let credentialsImported: CredentialImportSummary | undefined;
1033
-
1034
- if (validation.entries) {
1035
- const bundleCredentials = extractCredentialsFromBundle(
1036
- validation.entries,
1037
- validation.manifest!,
1038
- );
1039
- credentialsImported = await importBundleCredentialsIntoCes(
1040
- bundleCredentials,
1041
- result.report,
1042
- );
1043
- }
1044
-
1045
- // Reconcile cadet:* metadata against CES so the gateway's
1046
- // readServiceCredentials can still find platform identity values even
1047
- // if Django's post-hatch provisioning raced with the import.
1048
- await reconcileCadetMetadataFromCes(result.report);
1049
-
1050
- // Invalidate in-process config cache so imported settings.json takes effect
1051
- invalidateConfigCache();
1052
-
1053
- // Check whether the imported database contains migration checkpoints from
1054
- // a newer version. This is non-blocking — the import has already
1055
- // succeeded — but we surface a warning so the caller knows some data may
1056
- // not be fully compatible with this daemon's schema.
1057
- appendNewerMigrationWarningsIfAny(result.report);
1058
-
1059
- return importCommitSuccessResult(result.report, credentialsImported);
1060
- } catch (err) {
1061
- // Preserve typed RouteError instances (e.g. UnprocessableEntityError for
1062
- // version_incompatible, BadRequestError for validation_failed) — only
1063
- // wrap genuinely unexpected errors as 500 InternalError.
1064
- if (err instanceof RouteError) {
1065
- throw err;
1066
- }
1067
- log.error({ err }, "Unexpected error during import commit");
1068
- throw new InternalError(
1069
- err instanceof Error ? err.message : "Unexpected import error",
1070
- );
1071
- }
1072
- }
1073
-
1074
- // ---------------------------------------------------------------------------
1075
- // GCS URL import pipeline — shared by the URL-body branch of
1076
- // POST /v1/migrations/import and POST /v1/migrations/import-from-gcs.
1077
- // ---------------------------------------------------------------------------
1078
-
1079
- /** 60 minutes — matches the gateway's upstream fetch deadline. */
1080
- const URL_FETCH_TIMEOUT_MS = 60 * 60 * 1000;
1081
-
1082
- const MigrationImportUrlBody = z.object({ url: z.string().min(1) });
1083
-
1084
- const MigrationImportPathBody = z.object({ path: z.string().min(1) });
1085
-
1086
- const MigrationImportFromGcsBody = z.object({ bundle_url: z.string().url() });
1087
-
1088
- /**
1089
- * Marker attached to errors that originate from the upstream HTTP body
1090
- * stream (peer reset, abort mid-stream, DNS/transport failure after
1091
- * headers were received). The handler's catch/result-mapping path looks
1092
- * for this tag to return 502 `fetch_failed` instead of 500
1093
- * `extraction_failed` for truncated bodies, matching the OpenAPI
1094
- * contract.
1095
- */
1096
- const kFetchBodyError = Symbol.for("cadet.migrationImport.fetchBodyError");
1097
-
1098
- /**
1099
- * Sidecar flag on the wrapper PassThrough indicating that its upstream
1100
- * was torn down by a tagged fetch-body error. Checked after
1101
- * streamCommitImport returns — the importer preserves the error message
1102
- * in `result.reason = "extraction_failed"` but strips the tag.
1103
- */
1104
- const kFetchBodyTornDown = Symbol.for(
1105
- "cadet.migrationImport.fetchBodyTornDown",
1106
- );
1107
-
1108
- function tagFetchBodyError(err: NodeJS.ErrnoException): void {
1109
- (err as unknown as Record<symbol, boolean>)[kFetchBodyError] = true;
1110
- }
1111
-
1112
- function isFetchBodyError(err: unknown): boolean {
1113
- if (!err || typeof err !== "object") {
1114
- return false;
1115
- }
1116
- return (err as unknown as Record<symbol, boolean>)[kFetchBodyError] === true;
1117
- }
1118
-
1119
- function wasFetchBodyTornDown(stream: PassThrough): boolean {
1120
- return (
1121
- (stream as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] === true
1122
- );
1123
- }
1124
-
1125
- /**
1126
- * Test seam shared by the import AND export URL handlers. Defaults to
1127
- * `undefined`, which keeps the validator strict (GCS host, HTTPS only, no
1128
- * explicit port) — the only production configuration on a normal managed
1129
- * deployment. Tests override it via `_setUrlImportValidatorOptionsForTests`
1130
- * (e.g. to point the validator at a local HTTP server fixture).
1131
- *
1132
- * NOTE: this is intentionally NOT initialized from the environment. The
1133
- * `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS` env var is import-scoped and is
1134
- * applied only by `resolveImportValidatorOptions` — never on the export
1135
- * upload path, so a relaxed import allowlist can't widen where the
1136
- * assistant is willing to PUT a bundle (SSRF). Widening the export path is
1137
- * its own explicit operator opt-in via the separate
1138
- * `CADET_MIGRATION_EXPORT_ALLOWED_HOSTS` env var (local/minikube only,
1139
- * where the cloud serves plain-http SigV4 URLs from SeaweedFS) — see
1140
- * `exportValidatorOptions`.
1141
- */
1142
- let urlValidatorOptions: ValidateGcsSignedUrlOptions | undefined;
1143
-
1144
- /**
1145
- * Parse the comma-separated `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS` env value
1146
- * into `ValidateGcsSignedUrlOptions`. Returns `undefined` when the value is
1147
- * absent or contains no non-empty host entries, so the validator falls back
1148
- * to its strict production default.
1149
- *
1150
- * Exported for unit testing of the env-parsing behavior.
1151
- */
1152
- export function parseMigrationImportAllowedHostsEnv(
1153
- raw: string | undefined,
1154
- ): ValidateGcsSignedUrlOptions | undefined {
1155
- if (!raw) {
1156
- return undefined;
1157
- }
1158
- const allowedHosts = raw
1159
- .split(",")
1160
- .map((host) => host.trim())
1161
- .filter((host) => host.length > 0);
1162
- return allowedHosts.length > 0 ? { allowedHosts } : undefined;
1163
- }
1164
-
1165
- /**
1166
- * Resolve the validator options for the IMPORT URL handlers only.
1167
- *
1168
- * Precedence: an explicit test override (when set) wins; otherwise the
1169
- * env-derived import allowlist from `CADET_MIGRATION_IMPORT_ALLOWED_HOSTS`.
1170
- * The cloud injects that env var only where the bundle is served over
1171
- * plain http from a non-GCS host — e.g. local/minikube, where the signed
1172
- * bundle URL points at `http://host.docker.internal/...`. Supplying a
1173
- * non-default allowlist intentionally relaxes the validator's scheme check
1174
- * to also accept `http:` (and skips the explicit-port check) for those
1175
- * explicitly-allowlisted hosts only; every other host still requires https.
1176
- *
1177
- * The export path deliberately does NOT call this — see `urlValidatorOptions`.
1178
- *
1179
- * Exported for unit testing.
1180
- */
1181
- export function resolveImportValidatorOptions(
1182
- testOverride: ValidateGcsSignedUrlOptions | undefined,
1183
- rawEnv: string | undefined,
1184
- ): ValidateGcsSignedUrlOptions | undefined {
1185
- return testOverride ?? parseMigrationImportAllowedHostsEnv(rawEnv);
1186
- }
1187
-
1188
- /**
1189
- * Effective validator options for the IMPORT URL handlers, combining the
1190
- * test seam with the import-only env allowlist.
1191
- */
1192
- function importValidatorOptions(): ValidateGcsSignedUrlOptions | undefined {
1193
- return resolveImportValidatorOptions(
1194
- urlValidatorOptions,
1195
- process.env.CADET_MIGRATION_IMPORT_ALLOWED_HOSTS,
1196
- );
1197
- }
1198
-
1199
- /**
1200
- * Effective validator options for the EXPORT (upload URL) handler,
1201
- * combining the test seam with the export-only env allowlist from
1202
- * `CADET_MIGRATION_EXPORT_ALLOWED_HOSTS`.
1203
- *
1204
- * Deliberately a separate env var from the import allowlist: the export
1205
- * path controls where the assistant will PUT a full workspace bundle, so
1206
- * widening it must be its own operator decision, never a side effect of
1207
- * relaxing imports. The cloud injects it only where the signed upload
1208
- * URL is served over plain http from a non-GCS host (local/minikube
1209
- * SeaweedFS); it is unset in staging/prod, keeping the strict GCS-only
1210
- * default there.
1211
- */
1212
- function exportValidatorOptions(): ValidateGcsSignedUrlOptions | undefined {
1213
- return resolveImportValidatorOptions(
1214
- urlValidatorOptions,
1215
- process.env.CADET_MIGRATION_EXPORT_ALLOWED_HOSTS,
1216
- );
1217
- }
1218
-
1219
- /**
1220
- * Test-only: override the allowed-host list used by the URL-body import
1221
- * handler. Call with `undefined` (or no arguments) to reset to production
1222
- * defaults. This is intentionally not exported from the module's public
1223
- * surface — tests import it directly from this file.
1224
- */
1225
- export function _setUrlImportValidatorOptionsForTests(
1226
- options: ValidateGcsSignedUrlOptions | undefined,
1227
- ): void {
1228
- urlValidatorOptions = options;
1229
- }
1230
-
1231
- /**
1232
- * Successful outcome of `runGcsImport`. Mirrors the wire shape produced by
1233
- * `importCommitSuccessResponse` (report fields spread at the top level, with
1234
- * an optional `credentialsImported` summary alongside) so the same value can
1235
- * be serialized directly as a Response body OR stashed as an async-job
1236
- * `result` — both the sync endpoint and the async job-status endpoint then
1237
- * hand the CLI a single, identical `ImportResponse`-compatible shape.
1238
- */
1239
- export interface ImportSummary extends ImportCommitReport {
1240
- credentialsImported?: CredentialImportSummary;
1241
- }
1242
-
1243
- /**
1244
- * Structured error thrown by `runGcsImport`. Carries the information needed
1245
- * to reconstruct the URL-body handler's legacy Response shapes and for the
1246
- * async-job registry to map to `error.code`/`upstreamStatus`.
1247
- */
1248
- interface GcsImportErrorInit {
1249
- code:
1250
- | "invalid_url"
1251
- | "fetch_failed"
1252
- | "validation_failed"
1253
- | "extraction_failed"
1254
- | "version_incompatible"
1255
- | "write_failed";
1256
- message: string;
1257
- upstreamStatus?: number;
1258
- reason?: string;
1259
- errors?: Array<{ code: string; message: string; path?: string }>;
1260
- partial_report?: ImportCommitReport;
1261
- /** Populated for `version_incompatible` — mirrors the cloud's PR #5470
1262
- * response shape so the URL-body endpoint can return the same body. */
1263
- bundle_compat?: RuntimeCompatibility;
1264
- /** Populated for `version_incompatible`. */
1265
- runtime_version?: string;
1266
- }
1267
-
1268
- class GcsImportError extends Error {
1269
- public readonly code: GcsImportErrorInit["code"];
1270
- public readonly upstreamStatus?: number;
1271
- public readonly reason?: string;
1272
- public readonly errors?: GcsImportErrorInit["errors"];
1273
- public readonly partial_report?: ImportCommitReport;
1274
- public readonly bundle_compat?: RuntimeCompatibility;
1275
- public readonly runtime_version?: string;
1276
-
1277
- constructor(init: GcsImportErrorInit) {
1278
- super(init.message);
1279
- this.name = "GcsImportError";
1280
- this.code = init.code;
1281
- if (init.upstreamStatus !== undefined) {
1282
- this.upstreamStatus = init.upstreamStatus;
1283
- }
1284
- if (init.reason !== undefined) {
1285
- this.reason = init.reason;
1286
- }
1287
- if (init.errors !== undefined) {
1288
- this.errors = init.errors;
1289
- }
1290
- if (init.partial_report !== undefined) {
1291
- this.partial_report = init.partial_report;
1292
- }
1293
- if (init.bundle_compat !== undefined) {
1294
- this.bundle_compat = init.bundle_compat;
1295
- }
1296
- if (init.runtime_version !== undefined) {
1297
- this.runtime_version = init.runtime_version;
1298
- }
1299
- }
1300
- }
1301
-
1302
- /**
1303
- * Fetch a .vbundle from a signed GCS URL and commit it via the streaming
1304
- * importer. On success, returns an `ImportSummary` the caller can serialize
1305
- * into a Response or stash as an async-job `result`. On failure, throws a
1306
- * `GcsImportError`:
1307
- *
1308
- * - `invalid_url` → URL failed `validateGcsSignedUrl` (pre-fetch).
1309
- * - `fetch_failed` → upstream fetch error, non-2xx response, missing
1310
- * body, OR a mid-stream body teardown tagged via
1311
- * `kFetchBodyError`. `upstreamStatus` is populated
1312
- * for non-2xx responses.
1313
- * - `validation_failed` → the bundle failed schema/structural validation
1314
- * inside `streamCommitImport`; `errors` carries
1315
- * the per-issue list.
1316
- * - `extraction_failed` → bundle extraction threw (malformed archive, hash
1317
- * mismatch, etc.) that was NOT an upstream tear-
1318
- * down. `reason` carries the importer's string.
1319
- * - `write_failed` → post-extraction disk write error; `partial_report`
1320
- * is attached when the importer produced one.
1321
- *
1322
- * The signed URL is never echoed into errors or logs — only the extracted
1323
- * `host`/`path` are.
1324
- */
1325
- async function runGcsImport(
1326
- url: string,
1327
- _correlationId?: string,
1328
- ): Promise<ImportSummary> {
1329
- // ── 1. Validate the URL (defense-in-depth; never log the raw URL).
1330
- const validated = validateGcsSignedUrl(url, importValidatorOptions());
1331
- if (!validated.ok) {
1332
- log.warn({ reason: validated.reason }, "Rejected migration import URL");
1333
- throw new GcsImportError({
1334
- code: "invalid_url",
1335
- message: `Invalid URL: ${validated.reason}`,
1336
- reason: validated.reason,
1337
- });
1338
- }
1339
-
1340
- log.info(
1341
- { host: validated.host, path: validated.path },
1342
- "migration import from URL",
1343
- );
1344
-
1345
- const startedAt = Date.now();
1346
-
1347
- // ── 2. Fetch the URL ──────────────────────────────────────────────────
1348
- let upstream: Response;
1349
- try {
1350
- upstream = await fetch(url, {
1351
- signal: AbortSignal.timeout(URL_FETCH_TIMEOUT_MS),
1352
- // SSRF guard: `validateGcsSignedUrl` only vetted the initial URL.
1353
- // Default fetch behavior follows 3xx responses, which would let a
1354
- // validated `storage.googleapis.com` URL redirect to an arbitrary
1355
- // host and bypass the allowlist. Reject redirects so we only ever
1356
- // read bytes from the URL the caller handed us.
1357
- redirect: "error",
1358
- });
1359
- } catch (err) {
1360
- log.error(
1361
- {
1362
- host: validated.host,
1363
- path: validated.path,
1364
- err: err instanceof Error ? err.message : String(err),
1365
- },
1366
- "Failed to fetch migration import URL",
1367
- );
1368
- throw new GcsImportError({
1369
- code: "fetch_failed",
1370
- message: err instanceof Error ? err.message : String(err),
1371
- });
1372
- }
1373
-
1374
- if (!upstream.ok) {
1375
- log.error(
1376
- {
1377
- host: validated.host,
1378
- path: validated.path,
1379
- upstream_status: upstream.status,
1380
- },
1381
- "Migration import URL fetch returned non-2xx",
1382
- );
1383
- // Drain the body so the underlying socket can be released promptly.
1384
- try {
1385
- await upstream.body?.cancel();
1386
- } catch {
1387
- /* best effort */
1388
- }
1389
- throw new GcsImportError({
1390
- code: "fetch_failed",
1391
- message: `Upstream fetch returned ${upstream.status}`,
1392
- upstreamStatus: upstream.status,
1393
- });
1394
- }
1395
-
1396
- if (!upstream.body) {
1397
- log.error(
1398
- { host: validated.host, path: validated.path },
1399
- "Migration import URL fetch returned no body",
1400
- );
1401
- throw new GcsImportError({
1402
- code: "fetch_failed",
1403
- message: "Upstream fetch returned no body",
1404
- });
1405
- }
1406
-
1407
- // ── 3. Stream the response through the importer ──────────────────────
1408
- // Convert the WHATWG ReadableStream from fetch() into a Node Readable so
1409
- // the tar-stream / gunzip / hash-verifier pipeline inside
1410
- // streamCommitImport can consume it via `.pipe()`.
1411
- const upstreamNodeStream = Readable.fromWeb(
1412
- upstream.body as unknown as import("node:stream/web").ReadableStream<Uint8Array>,
1413
- );
1414
-
1415
- // Wrap the upstream stream in a PassThrough that tags any error bubbling
1416
- // from the upstream HTTP body (peer reset, abort mid-stream, etc.) with a
1417
- // known symbol. When that tagged error surfaces out of
1418
- // streamCommitImport's gunzip/tar pipeline, we can distinguish it from a
1419
- // legitimate bundle-format failure and map it to `fetch_failed` instead
1420
- // of `extraction_failed` — matching the OpenAPI contract for the URL
1421
- // body shape. We also propagate errors from the wrapper back to the
1422
- // upstream stream so its underlying connection is torn down cleanly.
1423
- //
1424
- // Bun's `Readable.fromWeb(fetchBody)` does NOT emit `'error'` when the
1425
- // TCP socket is torn down mid-response — it just emits `'close'` with
1426
- // no final `'end'`. We therefore track BOTH signals:
1427
- // • explicit `'error'` → tag the error, destroy the wrapper.
1428
- // • premature `'close'` → synthesize an error, tag it, destroy the
1429
- // wrapper. "Premature" = close fired without end first.
1430
- const taggedSource = new PassThrough();
1431
- let upstreamEnded = false;
1432
- // True once the importer (or any local consumer) initiates a teardown of
1433
- // `taggedSource`. The subsequent `close` on `upstreamNodeStream` is then a
1434
- // cascaded effect of our own teardown, NOT a real upstream failure — so
1435
- // we must NOT tag it as a fetch-body error, or local validation /
1436
- // extraction errors would be masked as fetch_failed.
1437
- let localTeardownInitiated = false;
1438
- upstreamNodeStream.on("end", () => {
1439
- upstreamEnded = true;
1440
- });
1441
- upstreamNodeStream.on("error", (err: NodeJS.ErrnoException) => {
1442
- tagFetchBodyError(err);
1443
- (taggedSource as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] =
1444
- true;
1445
- taggedSource.destroy(err);
1446
- });
1447
- upstreamNodeStream.on("close", () => {
1448
- if (upstreamEnded) {
1449
- return;
1450
- }
1451
- // A local teardown path closed us; don't treat this as an upstream
1452
- // failure. The real error (validation / extraction / hash mismatch) is
1453
- // already propagating through `streamCommitImport`'s result.
1454
- if (localTeardownInitiated) {
1455
- return;
1456
- }
1457
- const err = new Error(
1458
- "Upstream body stream closed before end",
1459
- ) as NodeJS.ErrnoException;
1460
- err.code = "ERR_UPSTREAM_BODY_CLOSED";
1461
- tagFetchBodyError(err);
1462
- (taggedSource as unknown as Record<symbol, boolean>)[kFetchBodyTornDown] =
1463
- true;
1464
- taggedSource.destroy(err);
1465
- });
1466
- upstreamNodeStream.pipe(taggedSource);
1467
- // Absorb stream errors on `taggedSource`. `streamCommitImport` does
1468
- // several `await`s (workspace recovery, temp-dir mkdir) before
1469
- // `parseVBundleStream(source)` attaches its own `source.on('error')`
1470
- // listener. If the upstream socket is destroyed during that window,
1471
- // the `upstreamNodeStream.on('close')` handler above calls
1472
- // `taggedSource.destroy(err)` with no listener attached yet and the
1473
- // error surfaces as unhandled. We don't need to act on it here — the
1474
- // `kFetchBodyTornDown` flag has already been latched on the stream,
1475
- // and the post-import branch below (`wasFetchBodyTornDown(taggedSource)`)
1476
- // maps the failure to `fetch_failed`. Register this absorber
1477
- // unconditionally so there is always at least one `'error'` listener
1478
- // on the wrapper for the lifetime of the stream.
1479
- taggedSource.on("error", () => {});
1480
- // Propagate wrapper teardown back to the upstream fetch body. When the
1481
- // streaming importer hits a validation/extraction error, it destroys
1482
- // `source` (which is `taggedSource`). Without this listener the
1483
- // `Readable.fromWeb(fetchBody)` stream would stay alive and continue
1484
- // buffering the remote response in the background until GC or the
1485
- // 60-minute timeout — a socket/bandwidth leak for any non-upstream error
1486
- // (malformed bundle, hash mismatch, size cap, etc.). We set
1487
- // `localTeardownInitiated` BEFORE destroying upstream so the resulting
1488
- // cascaded `close` on `upstreamNodeStream` isn't misclassified as a real
1489
- // upstream failure (which would return fetch_failed and mask the actual
1490
- // validation error).
1491
- taggedSource.on("close", () => {
1492
- if (!upstreamNodeStream.destroyed) {
1493
- localTeardownInitiated = true;
1494
- upstreamNodeStream.destroy();
1495
- }
1496
- });
1497
-
1498
- const pathResolver = new DefaultPathResolver(
1499
- getWorkspaceDir(),
1500
- getWorkspaceHooksDir(),
1501
- );
1502
-
1503
- // streamCommitImport does its own resetDb() internally before the atomic
1504
- // swap, so we don't need to call it here.
1505
- let result: ImportCommitResult;
1506
- // Track credential-import outcome for inclusion in the success response.
1507
- // The streaming importer invokes our callback only after the atomic swap,
1508
- // so filling this in here is safe.
1509
- let credentialsImported: CredentialImportSummary | undefined;
1510
- // Per-invocation warning collector — scoped to this request so concurrent
1511
- // URL imports can't trample each other's warnings.
1512
- const credentialImportWarningSink: CredentialWarningSink = { warnings: [] };
1513
-
1514
- try {
1515
- result = await streamCommitImport({
1516
- source: taggedSource,
1517
- pathResolver,
1518
- workspaceDir: getWorkspaceDir(),
1519
- importCredentials: async (bundleCredentials) => {
1520
- // We can't mutate `result.report.warnings` in place here — the
1521
- // streaming importer hasn't returned its report yet. Accumulate
1522
- // into a sidecar and merge into the final report below.
1523
- credentialsImported = await importBundleCredentialsIntoCes(
1524
- bundleCredentials,
1525
- credentialImportWarningSink,
1526
- );
1527
- },
1528
- });
1529
- } catch (err) {
1530
- if (isFetchBodyError(err)) {
1531
- log.error(
1532
- {
1533
- host: validated.host,
1534
- path: validated.path,
1535
- err: err instanceof Error ? err.message : String(err),
1536
- },
1537
- "Upstream body stream failed mid-import",
1538
- );
1539
- throw new GcsImportError({
1540
- code: "fetch_failed",
1541
- message: err instanceof Error ? err.message : String(err),
1542
- });
1543
- }
1544
- log.error(
1545
- {
1546
- host: validated.host,
1547
- path: validated.path,
1548
- err: err instanceof Error ? err.message : String(err),
1549
- },
1550
- "streamCommitImport threw during URL-body import",
1551
- );
1552
- throw new GcsImportError({
1553
- code: "extraction_failed",
1554
- message: err instanceof Error ? err.message : "Unexpected import error",
1555
- });
1556
- }
1557
-
1558
- if (!result.ok) {
1559
- // streamCommitImport swallows the raw cause and maps any
1560
- // non-validation throw to `extraction_failed`. If the cause was an
1561
- // upstream body failure that we tagged at the source, surface the
1562
- // tag through the result (the importer preserves the message) by
1563
- // detecting the latched flag on the wrapper stream.
1564
- if (wasFetchBodyTornDown(taggedSource)) {
1565
- log.error(
1566
- {
1567
- host: validated.host,
1568
- path: validated.path,
1569
- reason: result.reason,
1570
- },
1571
- "Upstream body stream failed mid-import (detected via result)",
1572
- );
1573
- throw new GcsImportError({
1574
- code: "fetch_failed",
1575
- message: "Upstream body stream failed mid-import",
1576
- });
1577
- }
1578
- log.warn(
1579
- {
1580
- host: validated.host,
1581
- path: validated.path,
1582
- reason: result.reason,
1583
- },
1584
- "streamCommitImport returned failure during URL-body import",
1585
- );
1586
- if (result.reason === "validation_failed") {
1587
- throw new GcsImportError({
1588
- code: "validation_failed",
1589
- message: "Bundle validation failed",
1590
- reason: result.reason,
1591
- errors: result.errors,
1592
- });
1593
- }
1594
- if (result.reason === "extraction_failed") {
1595
- throw new GcsImportError({
1596
- code: "extraction_failed",
1597
- message: result.message,
1598
- reason: result.reason,
1599
- });
1600
- }
1601
- if (result.reason === "version_incompatible") {
1602
- // Returned by commitImport / streamCommitImport when the runtime falls
1603
- // outside the bundle's compat range. The cloud-side gate is the
1604
- // primary check; this catches legacy bundles whose ExportJob row
1605
- // predates PR #5470 (compat columns NULL → platform gate skipped).
1606
- throw new GcsImportError({
1607
- code: "version_incompatible",
1608
- message: formatRuntimeCompatibilityMessage(
1609
- result.bundle_compat,
1610
- result.runtime_version,
1611
- ),
1612
- reason: result.reason,
1613
- bundle_compat: result.bundle_compat,
1614
- runtime_version: result.runtime_version,
1615
- });
1616
- }
1617
- // write_failed
1618
- throw new GcsImportError({
1619
- code: "write_failed",
1620
- message: result.message,
1621
- reason: result.reason,
1622
- partial_report: result.partial_report,
1623
- });
1624
- }
1625
-
1626
- // Merge any warnings accumulated by the credential-import callback into
1627
- // the final report.
1628
- if (credentialImportWarningSink.warnings.length > 0) {
1629
- result.report.warnings.push(...credentialImportWarningSink.warnings);
1630
- }
1631
-
1632
- // Reconcile cadet:* metadata against CES so the gateway's
1633
- // readServiceCredentials can still find platform identity values even
1634
- // if Django's post-hatch provisioning raced with the streaming import
1635
- // (its metadata upsert may have landed in the backup-dir copy that the
1636
- // swap pushed aside, while its CES write survived on the separate
1637
- // volume).
1638
- await reconcileCadetMetadataFromCes(result.report);
1639
-
1640
- // streamCommitImport already invalidated config + trust caches inside its
1641
- // post-swap cleanup. We only need to check whether the newly-imported DB
1642
- // carries migration checkpoints from a newer daemon version.
1643
- appendNewerMigrationWarningsIfAny(result.report);
1644
-
1645
- const elapsedMs = Date.now() - startedAt;
1646
- log.info(
1647
- {
1648
- host: validated.host,
1649
- path: validated.path,
1650
- files_written: result.report.summary.files_created,
1651
- bytes_written: result.report.files.reduce((n, f) => n + f.size, 0),
1652
- elapsed_ms: elapsedMs,
1653
- },
1654
- "Migration import from URL complete",
1655
- );
1656
-
1657
- return credentialsImported
1658
- ? { ...result.report, credentialsImported }
1659
- : { ...result.report };
1660
- }
1661
-
1662
- /**
1663
- * Dispatch a JSON import body to the URL-fetch or staged-path streamer.
1664
- * The body must include exactly one of `url` or `path`.
1665
- */
1666
- async function handleMigrationImportFromJson(
1667
- body: Record<string, unknown> | undefined,
1668
- ): Promise<unknown> {
1669
- const hasUrl = typeof body?.url === "string";
1670
- const hasPath = typeof body?.path === "string";
1671
- if (hasUrl && hasPath) {
1672
- throw new BadRequestError(
1673
- "Request body must include exactly one of url or path",
1674
- );
1675
- }
1676
- if (hasPath) {
1677
- return handleMigrationImportFromPath(body);
1678
- }
1679
- return handleMigrationImportFromUrl(body);
1680
- }
1681
-
1682
- async function handleMigrationImportFromPath(
1683
- body: Record<string, unknown> | undefined,
1684
- ): Promise<unknown> {
1685
- const parsed = MigrationImportPathBody.safeParse(body);
1686
- if (!parsed.success) {
1687
- throw new BadRequestError(
1688
- "Request body must be { path: string } with a non-empty path",
1689
- );
1690
- }
1691
-
1692
- let resolvedPath: string;
1693
- try {
1694
- resolvedPath = resolveStagedImportPath(parsed.data.path, getWorkspaceDir());
1695
- } catch (err) {
1696
- if (err instanceof StagedImportPathError) {
1697
- throw new BadRequestError(err.message);
1698
- }
1699
- throw err;
1700
- }
1701
-
1702
- const source = createReadStream(resolvedPath);
1703
- const pathResolver = new DefaultPathResolver(
1704
- getWorkspaceDir(),
1705
- getWorkspaceHooksDir(),
1706
- );
1707
- let credentialsImported: CredentialImportSummary | undefined;
1708
- const credentialImportWarningSink: CredentialWarningSink = { warnings: [] };
1709
-
1710
- try {
1711
- const result = await streamCommitImport({
1712
- source,
1713
- pathResolver,
1714
- workspaceDir: getWorkspaceDir(),
1715
- importCredentials: async (bundleCredentials) => {
1716
- credentialsImported = await importBundleCredentialsIntoCes(
1717
- bundleCredentials,
1718
- credentialImportWarningSink,
1719
- );
1720
- },
1721
- });
1722
-
1723
- if (!result.ok) {
1724
- throwImportCommitFailure(result);
1725
- }
1726
-
1727
- if (credentialImportWarningSink.warnings.length > 0) {
1728
- result.report.warnings.push(...credentialImportWarningSink.warnings);
1729
- }
1730
-
1731
- await reconcileCadetMetadataFromCes(result.report);
1732
- appendNewerMigrationWarningsIfAny(result.report);
1733
- return importCommitSuccessResult(result.report, credentialsImported);
1734
- } catch (err) {
1735
- if (err instanceof RouteError) {
1736
- throw err;
1737
- }
1738
- log.error({ err }, "Unexpected error during staged-path import");
1739
- throw new InternalError(
1740
- err instanceof Error ? err.message : "Unexpected import error",
1741
- );
1742
- } finally {
1743
- source.destroy();
1744
- }
1745
- }
1746
-
1747
- async function handleMigrationPreflightFromPath(
1748
- body: Record<string, unknown> | undefined,
1749
- ): Promise<unknown> {
1750
- const parsed = MigrationImportPathBody.safeParse(body);
1751
- if (!parsed.success) {
1752
- throw new BadRequestError(
1753
- "Request body must be { path: string } with a non-empty path",
1754
- );
1755
- }
1756
-
1757
- let resolvedPath: string;
1758
- try {
1759
- resolvedPath = resolveStagedImportPath(parsed.data.path, getWorkspaceDir());
1760
- } catch (err) {
1761
- if (err instanceof StagedImportPathError) {
1762
- throw new BadRequestError(err.message);
1763
- }
1764
- throw err;
1765
- }
1766
-
1767
- const source = createReadStream(resolvedPath);
1768
- try {
1769
- const entries = parseVBundleStream(source);
1770
- const first = await entries.next();
1771
- if (first.done) {
1772
- return {
1773
- can_import: false,
1774
- validation: {
1775
- is_valid: false as const,
1776
- errors: [
1777
- {
1778
- code: "empty_archive",
1779
- message: "Bundle archive is empty",
1780
- },
1781
- ],
1782
- },
1783
- };
1784
- }
1785
-
1786
- let manifest;
1787
- try {
1788
- ({ manifest } = await readAndValidateManifest(first.value));
1789
- } catch (err) {
1790
- if (err instanceof StreamingValidationError) {
1791
- return {
1792
- can_import: false,
1793
- validation: {
1794
- is_valid: false as const,
1795
- errors: [
1796
- {
1797
- code: err.code,
1798
- message: err.message,
1799
- ...(err.archivePath !== undefined && { path: err.archivePath }),
1800
- },
1801
- ],
1802
- },
1803
- };
1804
- }
1805
- throw err;
1806
- }
1807
-
1808
- for await (const entry of entries) {
1809
- entry.body.resume();
1810
- }
1811
-
1812
- const pathResolver = new DefaultPathResolver(
1813
- getWorkspaceDir(),
1814
- getWorkspaceHooksDir(),
1815
- );
1816
- return analyzeImport({ manifest, pathResolver });
1817
- } catch (err) {
1818
- if (err instanceof RouteError) {
1819
- throw err;
1820
- }
1821
- log.error({ err }, "Unexpected error during staged-path preflight");
1822
- throw new InternalError(
1823
- err instanceof Error ? err.message : "Unexpected import preflight error",
1824
- );
1825
- } finally {
1826
- source.destroy();
1827
- }
1828
- }
1829
-
1830
- /**
1831
- * Handle a JSON `{ "url": "..." }` body on POST /v1/migrations/import.
1832
- *
1833
- * Thin wrapper around `runGcsImport` that preserves the legacy synchronous
1834
- * Response shapes. `handleMigrationImportFromGcs` below uses the same helper
1835
- * asynchronously via the migration-job registry.
1836
- */
1837
- async function handleMigrationImportFromUrl(
1838
- body: Record<string, unknown> | undefined,
1839
- ): Promise<unknown> {
1840
- const parsed = MigrationImportUrlBody.safeParse(body);
1841
- if (!parsed.success) {
1842
- throw new BadRequestError(
1843
- "Request body must be { url: string } with a non-empty url",
1844
- );
1845
- }
1846
-
1847
- try {
1848
- const summary = await runGcsImport(parsed.data.url);
1849
- const { credentialsImported, ...report } = summary;
1850
- return importCommitSuccessResult(report, credentialsImported);
1851
- } catch (err) {
1852
- throwGcsImportError(err);
1853
- }
1854
- }
1855
-
1856
- /**
1857
- * Map a `runGcsImport` error (or any other thrown value) to a thrown
1858
- * RouteError subclass or a plain-object error body. Always throws —
1859
- * callers should invoke this in a catch block.
1860
- */
1861
- function throwGcsImportError(err: unknown): never {
1862
- if (err instanceof GcsImportError) {
1863
- if (err.code === "invalid_url") {
1864
- throw new BadRequestError(err.message);
1865
- }
1866
- if (err.code === "fetch_failed") {
1867
- throw new BadGatewayError(
1868
- err.upstreamStatus
1869
- ? `Upstream fetch returned ${err.upstreamStatus}`
1870
- : err.message,
1871
- );
1872
- }
1873
- if (err.code === "validation_failed") {
1874
- // Validation failure is not an HTTP error — return structured body
1875
- // with 200 (same as raw-bytes validate path).
1876
- throw new BadRequestError(
1877
- JSON.stringify({
1878
- success: false,
1879
- reason: "validation_failed",
1880
- errors: err.errors ?? [],
1881
- }),
1882
- );
1883
- }
1884
- if (err.code === "version_incompatible") {
1885
- // 422 (not 500) — the bundle is structurally valid but cannot be
1886
- // imported on this runtime. Body mirrors the cloud's PR #5470
1887
- // response shape.
1888
- throw new UnprocessableEntityError(err.message, {
1889
- reason: "version_incompatible" as const,
1890
- ...(err.bundle_compat !== undefined && {
1891
- bundle_compat: err.bundle_compat,
1892
- }),
1893
- ...(err.runtime_version !== undefined && {
1894
- runtime_version: err.runtime_version,
1895
- }),
1896
- });
1897
- }
1898
- if (err.code === "extraction_failed") {
1899
- throw new InternalError(err.message);
1900
- }
1901
- // write_failed
1902
- throw new InternalError(err.message);
1903
- }
1904
-
1905
- log.error({ err }, "Unexpected error from runGcsImport");
1906
- throw new InternalError(
1907
- err instanceof Error ? err.message : "Unexpected import error",
1908
- );
1909
- }
1910
-
1911
- /**
1912
- * POST /v1/migrations/import-from-gcs
1913
- *
1914
- * Kick off an async bundle import from a signed GCS URL. Returns 202 with a
1915
- * `job_id` the caller can poll via `GET /v1/migrations/jobs/:job_id`
1916
- * (PR 4). 409 if another import is already pending or running.
1917
- *
1918
- * Auth: Requires settings.write scope. Allowed for actor, svc_gateway, svc_daemon, local.
1919
- */
1920
- export async function handleMigrationImportFromGcs({ body }: RouteHandlerArgs) {
1921
- const parsed = MigrationImportFromGcsBody.safeParse(body);
1922
- if (!parsed.success) {
1923
- throw new BadRequestError(
1924
- "Request body must be { bundle_url: string } with a valid URL",
1925
- );
1926
- }
1927
-
1928
- const { bundle_url } = parsed.data;
1929
-
1930
- // Synchronously validate the GCS URL before consuming the single
1931
- // in-flight import slot.
1932
- const validated = validateGcsSignedUrl(bundle_url, importValidatorOptions());
1933
- if (!validated.ok) {
1934
- log.warn(
1935
- { reason: validated.reason },
1936
- "Rejected migration import-from-gcs bundle URL",
1937
- );
1938
- throw new RouteError(
1939
- `Invalid bundle URL: ${validated.reason}`,
1940
- "invalid_bundle_url",
1941
- 400,
1942
- );
1943
- }
1944
-
1945
- try {
1946
- const job = migrationJobs.startJob("import", async (jobRecord) =>
1947
- trackDaemonActivity({ kind: "vbundle_import" }, () =>
1948
- runGcsImport(bundle_url, jobRecord.id),
1949
- ),
1950
- );
1951
- return {
1952
- job_id: job.id,
1953
- status: "pending" as const,
1954
- type: "import" as const,
1955
- };
1956
- } catch (err) {
1957
- if (err instanceof JobAlreadyInProgressError) {
1958
- throw new RouteError(
1959
- `Import already in progress: ${err.existingJobId}`,
1960
- "import_in_progress",
1961
- 409,
1962
- );
1963
- }
1964
- log.error({ err }, "Unexpected error scheduling import-from-gcs job");
1965
- throw new InternalError(
1966
- err instanceof Error ? err.message : "Unexpected import error",
1967
- );
1968
- }
1969
- }
1970
-
1971
- // ---------------------------------------------------------------------------
1972
- // Shared helpers for raw-bytes and URL paths
1973
- // ---------------------------------------------------------------------------
1974
- // Preflight-from-GCS: dry-run analysis using a signed GCS download URL
1975
- // ---------------------------------------------------------------------------
1976
-
1977
- /**
1978
- * POST /v1/migrations/preflight-from-gcs
1979
- *
1980
- * Dry-run import analysis using a signed GCS download URL. Fetches the
1981
- * bundle from GCS, validates it, and returns the same preflight report as
1982
- * /v1/migrations/import-preflight — without writing anything to disk.
1983
- *
1984
- * This allows the CLI to run `cadet teleport --dry-run` against local and
1985
- * docker targets (which do not have a direct-upload preflight path like the
1986
- * platform does) by reusing the same GCS-mediated bundle it would use for
1987
- * a real import.
1988
- *
1989
- * Auth: Requires settings.write scope.
1990
- */
1991
- export async function handleMigrationPreflightFromGcs({
1992
- body,
1993
- }: RouteHandlerArgs) {
1994
- const parsed = z.object({ bundle_url: z.string().url() }).safeParse(body);
1995
- if (!parsed.success) {
1996
- throw new BadRequestError(
1997
- "Request body must be { bundle_url: string } with a valid URL",
1998
- );
1999
- }
2000
-
2001
- const { bundle_url } = parsed.data;
2002
- const validated = validateGcsSignedUrl(bundle_url, importValidatorOptions());
2003
- if (!validated.ok) {
2004
- log.warn(
2005
- { reason: validated.reason },
2006
- "Rejected preflight-from-gcs bundle URL",
2007
- );
2008
- throw new RouteError(
2009
- `Invalid bundle URL: ${validated.reason}`,
2010
- "invalid_bundle_url",
2011
- 400,
2012
- );
2013
- }
2014
-
2015
- let upstream: Response;
2016
- try {
2017
- upstream = await fetch(bundle_url, {
2018
- signal: AbortSignal.timeout(URL_FETCH_TIMEOUT_MS),
2019
- // Reject redirects: the URL was already validated against the GCS
2020
- // allowlist and we must not follow it to an arbitrary host.
2021
- redirect: "error",
2022
- });
2023
- } catch (err) {
2024
- log.error(
2025
- {
2026
- host: validated.host,
2027
- path: validated.path,
2028
- err: err instanceof Error ? err.message : String(err),
2029
- },
2030
- "Failed to fetch preflight-from-gcs bundle URL",
2031
- );
2032
- throw new InternalError(
2033
- `Failed to fetch bundle: ${err instanceof Error ? err.message : String(err)}`,
2034
- );
2035
- }
2036
-
2037
- if (!upstream.ok) {
2038
- log.error(
2039
- {
2040
- host: validated.host,
2041
- path: validated.path,
2042
- upstream_status: upstream.status,
2043
- },
2044
- "preflight-from-gcs bundle URL returned non-2xx",
2045
- );
2046
- throw new InternalError(
2047
- `Bundle URL returned ${upstream.status} — the signed URL may have expired.`,
2048
- );
2049
- }
2050
-
2051
- const bytes = new Uint8Array(await upstream.arrayBuffer());
2052
- const validationResult = validateVBundle(bytes, {
2053
- enforceStreamingLimits: true,
2054
- });
2055
-
2056
- if (!validationResult.is_valid || !validationResult.manifest) {
2057
- return {
2058
- can_import: false,
2059
- validation: {
2060
- is_valid: false as const,
2061
- errors: validationResult.errors,
2062
- },
2063
- };
2064
- }
2065
-
2066
- const pathResolver = new DefaultPathResolver(
2067
- getWorkspaceDir(),
2068
- getWorkspaceHooksDir(),
2069
- );
2070
-
2071
- try {
2072
- return analyzeImport({ manifest: validationResult.manifest, pathResolver });
2073
- } catch (err) {
2074
- log.error({ err }, "Unexpected error during preflight-from-gcs analysis");
2075
- throw new InternalError(
2076
- err instanceof Error
2077
- ? err.message
2078
- : "Unexpected preflight-from-gcs error",
2079
- );
2080
- }
2081
- }
2082
-
2083
- // ---------------------------------------------------------------------------
2084
-
2085
- interface CredentialImportSummary {
2086
- total: number;
2087
- succeeded: number;
2088
- failed: number;
2089
- failedAccounts: string[];
2090
- skippedCloud: number;
2091
- }
2092
-
2093
- /**
2094
- * Minimal surface the credential-import helper needs to stash warnings —
2095
- * either a full `ImportCommitReport` (raw-bytes path, after commitImport
2096
- * returns) or an ephemeral per-request collector (streaming path, where the
2097
- * report doesn't exist yet when the callback fires).
2098
- */
2099
- interface CredentialWarningSink {
2100
- warnings: string[];
2101
- }
2102
-
2103
- /**
2104
- * Filter cloud-identity (cadet:*) credentials out of the bundle, push
2105
- * user credentials into CES via `bulkSetSecureKeysAsync`, and return a
2106
- * structured summary. Never throws — CES failures become report warnings.
2107
- */
2108
- async function importBundleCredentialsIntoCes(
2109
- bundleCredentials: Array<{ account: string; value: string }>,
2110
- warningSink: CredentialWarningSink,
2111
- ): Promise<CredentialImportSummary | undefined> {
2112
- // Filter out cloud-identity credentials (cadet:*) — these are
2113
- // environment-specific and must not overwrite the target's own identity.
2114
- const userCredentials = bundleCredentials.filter(
2115
- (c) => !c.account.startsWith(PLATFORM_CREDENTIAL_PREFIX),
2116
- );
2117
- const skippedCloud = bundleCredentials.length - userCredentials.length;
2118
- if (skippedCloud > 0) {
2119
- log.info(`Skipped ${skippedCloud} cloud credential(s) from import`);
2120
- }
2121
-
2122
- if (userCredentials.length === 0) {
2123
- if (skippedCloud > 0) {
2124
- // All credentials in the bundle were cloud credentials — report
2125
- // the skip count even though nothing was sent to CES.
2126
- return {
2127
- total: 0,
2128
- succeeded: 0,
2129
- failed: 0,
2130
- failedAccounts: [],
2131
- skippedCloud,
2132
- };
2133
- }
2134
- return undefined;
2135
- }
2136
-
2137
- try {
2138
- const credResults = await bulkSetSecureKeysAsync(userCredentials);
2139
- const failedResults = credResults.filter((r) => !r.ok);
2140
- if (failedResults.length > 0) {
2141
- log.warn(
2142
- { failed: failedResults.map((f) => f.account) },
2143
- "Some credentials failed to import",
2144
- );
2145
- }
2146
- log.info(
2147
- { total: userCredentials.length, failed: failedResults.length },
2148
- "Credential import complete",
2149
- );
2150
- const succeeded = userCredentials.length - failedResults.length;
2151
- if (failedResults.length > 0) {
2152
- warningSink.warnings.push(
2153
- `Imported ${succeeded} credential(s), ${failedResults.length} failed`,
2154
- );
2155
- }
2156
- return {
2157
- total: userCredentials.length,
2158
- succeeded,
2159
- failed: failedResults.length,
2160
- failedAccounts: failedResults.map((f) => f.account),
2161
- skippedCloud,
2162
- };
2163
- } catch (err) {
2164
- log.warn({ err }, "Credential import failed entirely");
2165
- warningSink.warnings.push(
2166
- `Credential import failed: ${err instanceof Error ? err.message : String(err)}`,
2167
- );
2168
- return {
2169
- total: userCredentials.length,
2170
- succeeded: 0,
2171
- failed: userCredentials.length,
2172
- failedAccounts: userCredentials.map((c) => c.account),
2173
- skippedCloud,
2174
- };
2175
- }
2176
- }
2177
-
2178
- /**
2179
- * Append a warning to `report` when the newly-imported database contains
2180
- * migration checkpoints from a daemon version newer than this one. Silent
2181
- * on any validation error — the import has already succeeded.
2182
- *
2183
- * Gated on the report's own file counts: if the import didn't create or
2184
- * overwrite any workspace files (no-swap success — e.g. credentials-only
2185
- * bundle, all-skipped legacy bundle), the live DB is unchanged and any
2186
- * "newer migrations" detected there came from the existing workspace,
2187
- * NOT from the imported bundle. Attributing them to the bundle would be a
2188
- * false positive, so skip the check entirely in that case.
2189
- */
2190
- function appendNewerMigrationWarningsIfAny(report: ImportCommitReport): void {
2191
- if (report.summary.files_created + report.summary.files_overwritten === 0) {
2192
- return;
2193
- }
2194
- try {
2195
- const migrationValidation = validateMigrationState(getDb(), migrationSteps);
2196
- if (migrationValidation.unknownCheckpoints.length > 0) {
2197
- report.warnings.push(
2198
- `Imported data contains ${migrationValidation.unknownCheckpoints.length} migration(s) from a newer version. Some data may not be fully compatible.`,
2199
- );
2200
- }
2201
- } catch {
2202
- // Don't fail the import if validation itself errors
2203
- }
2204
- }
2205
-
2206
- /**
2207
- * Build a success result from an ImportCommitReport.
2208
- */
2209
- function importCommitSuccessResult(
2210
- report: ImportCommitReport,
2211
- credentialsImported: CredentialImportSummary | undefined,
2212
- ): unknown {
2213
- return {
2214
- ...report,
2215
- ...(credentialsImported ? { credentialsImported } : {}),
2216
- };
2217
- }
2218
-
2219
- /**
2220
- * Map an `ImportCommitResult` failure to a thrown error or a plain-object
2221
- * error body. Status codes and body shapes are part of the public contract
2222
- * and must remain stable.
2223
- */
2224
- function throwImportCommitFailure(
2225
- result: Extract<ImportCommitResult, { ok: false }>,
2226
- ): never {
2227
- if (result.reason === "validation_failed") {
2228
- // Validation failure uses 400 — structured body with error details
2229
- throw new BadRequestError(
2230
- JSON.stringify({
2231
- success: false,
2232
- reason: "validation_failed",
2233
- errors: result.errors,
2234
- }),
2235
- );
2236
- }
2237
-
2238
- if (result.reason === "extraction_failed") {
2239
- throw new InternalError(result.message);
2240
- }
2241
-
2242
- if (result.reason === "version_incompatible") {
2243
- // Returned by commitImport / streamCommitImport when the runtime falls
2244
- // outside the bundle's compat range. The cloud-side gate is the
2245
- // primary check; this catches legacy bundles whose ExportJob row
2246
- // predates PR #5470 (compat columns NULL → platform gate skipped).
2247
- //
2248
- // 422 (not 500) — the bundle is structurally valid but cannot be
2249
- // imported on this runtime; the caller can act on it (upgrade the
2250
- // runtime, choose a different bundle). Body mirrors the cloud's
2251
- // PR #5470 response shape.
2252
- throw new UnprocessableEntityError(
2253
- formatRuntimeCompatibilityMessage(
2254
- result.bundle_compat,
2255
- result.runtime_version,
2256
- ),
2257
- {
2258
- reason: "version_incompatible" as const,
2259
- bundle_compat: result.bundle_compat,
2260
- runtime_version: result.runtime_version,
2261
- },
2262
- );
2263
- }
2264
-
2265
- // write_failed
2266
- throw new InternalError(result.message);
2267
- }
2268
-
2269
- // ---------------------------------------------------------------------------
2270
- // GET /v1/migrations/jobs/:job_id
2271
- // ---------------------------------------------------------------------------
2272
-
2273
- /**
2274
- * GET /v1/migrations/jobs/:job_id
2275
- *
2276
- * Returns the current status of a migration job tracked by
2277
- * `MigrationJobRegistry`. The response shape is a discriminated union on
2278
- * `status`:
2279
- *
2280
- * - `{ job_id, type, status: "processing" }`
2281
- * Covers both the internal `pending` and `running` states — collapsed
2282
- * into a single wire value to match the cloud's transport shape used
2283
- * by `ExportStatusProcessingSerializer` / `ImportStatusProcessingSerializer`.
2284
- * - `{ job_id, type, status: "complete", result }`
2285
- * - `{ job_id, type, status: "failed", error, error_code, upstream_status? }`
2286
- *
2287
- * 404 `{ error: { code: "job_not_found" } }` when no job matches the id.
2288
- */
2289
- export async function handleMigrationJobStatus({
2290
- pathParams,
2291
- }: RouteHandlerArgs) {
2292
- const jobId = pathParams?.job_id;
2293
- if (!jobId) {
2294
- throw new BadRequestError("Missing job_id path parameter");
2295
- }
2296
-
2297
- const job = migrationJobs.getJob(jobId);
2298
- if (job === null) {
2299
- throw new NotFoundError("Job not found");
2300
- }
2301
-
2302
- if (job.status === "complete") {
2303
- return {
2304
- job_id: job.id,
2305
- type: job.type,
2306
- status: "complete",
2307
- result: job.result,
2308
- };
2309
- }
2310
-
2311
- if (job.status === "failed") {
2312
- const error = job.error;
2313
- const result: Record<string, unknown> = {
2314
- job_id: job.id,
2315
- type: job.type,
2316
- status: "failed",
2317
- error: error?.message ?? "unknown",
2318
- error_code: error?.code ?? "unknown",
2319
- };
2320
- if (error?.upstreamStatus !== undefined) {
2321
- result.upstream_status = error.upstreamStatus;
2322
- }
2323
- return result;
2324
- }
2325
-
2326
- // pending or running — collapse to the cloud's "processing" wire value.
2327
- return {
2328
- job_id: job.id,
2329
- type: job.type,
2330
- status: "processing",
2331
- };
2332
- }
2333
-
2334
- // ---------------------------------------------------------------------------
2335
- // Route definitions
2336
- // ---------------------------------------------------------------------------
2337
-
2338
- export const ROUTES: RouteDefinition[] = [
2339
- {
2340
- operationId: "migrations_validate_post",
2341
- endpoint: "migrations/validate",
2342
- method: "POST",
2343
- policy: {
2344
- requiredScopes: ["settings.read"],
2345
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2346
- },
2347
- summary: "Validate a .vbundle archive",
2348
- description:
2349
- "Upload a .vbundle archive for validation. Accepts raw binary or multipart form data.",
2350
- tags: ["migrations"],
2351
- responseBody: z.object({
2352
- is_valid: z.boolean(),
2353
- errors: z.array(z.unknown()),
2354
- manifest: z.object({}).passthrough(),
2355
- }),
2356
- handler: handleMigrationValidate,
2357
- },
2358
- {
2359
- operationId: "migrations_export_post",
2360
- endpoint: "migrations/export",
2361
- method: "POST",
2362
- policy: {
2363
- requiredScopes: ["settings.write"],
2364
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2365
- },
2366
- summary: "Export a .vbundle archive",
2367
- description:
2368
- "Generate and download a .vbundle archive of the assistant's data. Optional JSON body for metadata.",
2369
- tags: ["migrations"],
2370
- requestBody: z.object({
2371
- description: z.string().describe("Human-readable export description"),
2372
- }),
2373
- handler: handleMigrationExport,
2374
- },
2375
- {
2376
- operationId: "migrations_importpreflight_post",
2377
- endpoint: "migrations/import-preflight",
2378
- method: "POST",
2379
- policy: {
2380
- requiredScopes: ["settings.write"],
2381
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2382
- },
2383
- summary: "Dry-run import analysis",
2384
- description:
2385
- "Validate a .vbundle archive and return a report of what would change on import without modifying data. Accepts raw bytes, multipart form data, or JSON `{ path }` pointing at a file staged under the workspace `.restore-staging` directory.",
2386
- tags: ["migrations"],
2387
- requestBody: z.object({
2388
- path: z
2389
- .string()
2390
- .min(1)
2391
- .optional()
2392
- .describe(
2393
- "Workspace-relative or absolute path to a staged .vbundle under .restore-staging (JSON body path only).",
2394
- ),
2395
- }),
2396
- responseBody: z.object({
2397
- can_import: z.boolean(),
2398
- summary: z.object({}).passthrough(),
2399
- files: z.array(z.unknown()),
2400
- conflicts: z.array(z.unknown()),
2401
- manifest: z.object({}).passthrough(),
2402
- }),
2403
- handler: handleMigrationImportPreflight,
2404
- },
2405
- {
2406
- operationId: "migrations_import_post",
2407
- endpoint: "migrations/import",
2408
- method: "POST",
2409
- policy: {
2410
- requiredScopes: ["settings.write"],
2411
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2412
- },
2413
- summary: "Import a .vbundle archive",
2414
- description:
2415
- "Commit a .vbundle archive import to disk — destructive. Accepts the bundle as raw bytes (application/octet-stream), multipart/form-data, a JSON body with `{ url }` carrying a signed URL the daemon fetches, or a JSON body with `{ path }` pointing at a file staged under the workspace `.restore-staging` directory.",
2416
- tags: ["migrations"],
2417
- requestBody: z.object({
2418
- url: z
2419
- .string()
2420
- .url()
2421
- .optional()
2422
- .describe(
2423
- "A signed GCS URL pointing to the .vbundle archive (JSON body path only).",
2424
- ),
2425
- path: z
2426
- .string()
2427
- .min(1)
2428
- .optional()
2429
- .describe(
2430
- "Workspace-relative or absolute path to a staged .vbundle under .restore-staging (JSON body path only).",
2431
- ),
2432
- }),
2433
- additionalResponses: {
2434
- "502": {
2435
- description: "Upstream fetch failed (URL body only).",
2436
- },
2437
- },
2438
- responseBody: z.object({
2439
- success: z.boolean(),
2440
- summary: z.object({}).passthrough(),
2441
- files: z.array(z.unknown()),
2442
- manifest: z.object({}).passthrough(),
2443
- warnings: z.array(z.unknown()),
2444
- }),
2445
- handler: (args) =>
2446
- trackDaemonActivity({ kind: "vbundle_import" }, () =>
2447
- handleMigrationImport(args),
2448
- ),
2449
- },
2450
- {
2451
- operationId: "migrations_exporttogcs_post",
2452
- endpoint: "migrations/export-to-gcs",
2453
- method: "POST",
2454
- policy: {
2455
- requiredScopes: ["settings.write"],
2456
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2457
- },
2458
- summary: "Start an async export streamed to a GCS signed URL",
2459
- description:
2460
- "Kick off a background export job that PUTs a freshly-built .vbundle archive to the supplied GCS signed URL. Returns 202 with a job_id the caller can poll via the job-status endpoint. Fails fast with 409 if another export job is already pending or running.",
2461
- tags: ["migrations"],
2462
- requestBody: z.object({
2463
- upload_url: z
2464
- .string()
2465
- .url()
2466
- .describe("Signed GCS PUT URL that receives the exported bundle."),
2467
- description: z
2468
- .string()
2469
- .optional()
2470
- .describe("Human-readable export description."),
2471
- profile: z
2472
- .enum(["migration", "debug"])
2473
- .optional()
2474
- .describe(
2475
- "Export profile. 'migration' (default) builds a teleport bundle. 'debug' builds a bundle for Cadet staff to open on a debug clone: no credentials, plus the gateway's database and logs under gateway/.",
2476
- ),
2477
- }),
2478
- responseStatus: "202",
2479
- responseBody: z.object({
2480
- job_id: z.string(),
2481
- status: z.literal("pending"),
2482
- type: z.literal("export"),
2483
- }),
2484
- handler: handleMigrationExportToGcs,
2485
- },
2486
- {
2487
- operationId: "migrations_importfromgcs_post",
2488
- endpoint: "migrations/import-from-gcs",
2489
- method: "POST",
2490
- policy: {
2491
- requiredScopes: ["settings.write"],
2492
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2493
- },
2494
- summary: "Start an async .vbundle import from a signed GCS URL",
2495
- description:
2496
- "Schedule a background import job that fetches the bundle at `bundle_url` and streams it through the importer. Returns 202 with a `job_id`; poll `GET /v1/migrations/jobs/{job_id}` for status. 409 if another import is already in flight.",
2497
- tags: ["migrations"],
2498
- requestBody: z.object({
2499
- bundle_url: z.string().url(),
2500
- }),
2501
- responseStatus: "202",
2502
- responseBody: z.object({
2503
- job_id: z.string(),
2504
- status: z.literal("pending"),
2505
- type: z.literal("import"),
2506
- }),
2507
- additionalResponses: {
2508
- "409": {
2509
- description: "Another import job is already pending or running.",
2510
- },
2511
- },
2512
- handler: handleMigrationImportFromGcs,
2513
- },
2514
- {
2515
- operationId: "migrations_preflightfromgcs_post",
2516
- endpoint: "migrations/preflight-from-gcs",
2517
- method: "POST",
2518
- policy: {
2519
- requiredScopes: ["settings.write"],
2520
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2521
- },
2522
- summary: "Dry-run import analysis from a signed GCS URL",
2523
- description:
2524
- "Fetch a .vbundle archive from a signed GCS download URL and return a preflight report — what would change if the bundle were imported — without writing anything to disk. Enables `cadet teleport --dry-run` against local and docker targets.",
2525
- tags: ["migrations"],
2526
- requestBody: z.object({
2527
- bundle_url: z
2528
- .string()
2529
- .url()
2530
- .describe("Signed GCS GET URL for the bundle."),
2531
- }),
2532
- responseBody: z.object({
2533
- can_import: z.boolean(),
2534
- summary: z.object({}).passthrough().optional(),
2535
- files: z.array(z.unknown()).optional(),
2536
- conflicts: z.array(z.unknown()).optional(),
2537
- validation: z.object({}).passthrough().optional(),
2538
- }),
2539
- handler: handleMigrationPreflightFromGcs,
2540
- },
2541
- {
2542
- operationId: "migrations_jobs_by_job_id_get",
2543
- endpoint: "migrations/jobs/:job_id",
2544
- method: "GET",
2545
- policy: {
2546
- requiredScopes: ["settings.read"],
2547
- allowedPrincipalTypes: ACTOR_PRINCIPALS,
2548
- },
2549
- summary: "Get migration job status",
2550
- description:
2551
- "Return the current status of an async migration job (export or import). The response discriminates on `status`: `processing` (pending or running), `complete` (with `result`), or `failed` (with `error`, `error_code`, optional `upstream_status`).",
2552
- tags: ["migrations"],
2553
- pathParams: [
2554
- { name: "job_id", description: "The migration job ID to query." },
2555
- ],
2556
- responseBody: z.discriminatedUnion("status", [
2557
- z.object({
2558
- job_id: z.string(),
2559
- type: z.enum(["export", "import"]),
2560
- status: z.literal("processing"),
2561
- }),
2562
- z.object({
2563
- job_id: z.string(),
2564
- type: z.enum(["export", "import"]),
2565
- status: z.literal("complete"),
2566
- result: z.unknown(),
2567
- }),
2568
- z.object({
2569
- job_id: z.string(),
2570
- type: z.enum(["export", "import"]),
2571
- status: z.literal("failed"),
2572
- error: z.string(),
2573
- error_code: z.string(),
2574
- upstream_status: z.number().int().optional(),
2575
- }),
2576
- ]),
2577
- additionalResponses: {
2578
- "404": {
2579
- description: "No job matches the given id.",
2580
- },
2581
- },
2582
- handler: handleMigrationJobStatus,
2583
- },
2584
- ];