@mycadet/assistant 0.12.10 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  2. package/node_modules/@mycadet/ces-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  3. package/node_modules/@mycadet/ces-client/src/http-credentials.ts +18 -1
  4. package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  5. package/node_modules/@mycadet/gateway-client/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  6. package/node_modules/@mycadet/gateway-client/src/gateway-ipc-contracts.ts +0 -20
  7. package/node_modules/@mycadet/service-contracts/src/__tests__/ingress.test.ts +10 -10
  8. package/node_modules/@mycadet/service-contracts/src/ingress.ts +3 -3
  9. package/openapi.yaml +2 -708
  10. package/package.json +2 -3
  11. package/scripts/sync-web-search-catalog.ts +1 -2
  12. package/src/__tests__/cloud-profile-guard.test.ts +41 -5
  13. package/src/__tests__/config-schema.test.ts +6 -38
  14. package/src/__tests__/conversation-queue.test.ts +45 -6
  15. package/src/__tests__/db-readiness-http-gate.test.ts +0 -6
  16. package/src/__tests__/persist-onboarding-artifacts.test.ts +0 -1
  17. package/src/__tests__/persona-resolver.test.ts +0 -51
  18. package/src/__tests__/provider-cloud-proxy-integration.test.ts +2 -110
  19. package/src/__tests__/run-due-schedules.test.ts +0 -4
  20. package/src/__tests__/schedule-store.test.ts +0 -12
  21. package/src/__tests__/stt-stream-session.test.ts +4 -4
  22. package/src/__tests__/telephony-tts-guard.test.ts +2 -50
  23. package/src/__tests__/voice-config-update.test.ts +12 -167
  24. package/src/__tests__/web-search-catalog-parity.test.ts +0 -2
  25. package/src/__tests__/workspace-migration-130-speech-mode-to-provider.test.ts +9 -7
  26. package/src/__tests__/workspace-migration-132-web-search-mode-to-provider.test.ts +6 -6
  27. package/src/calls/__tests__/voice-session-bridge.test.ts +1 -4
  28. package/src/calls/resolve-call-tts-provider.ts +6 -25
  29. package/src/calls/telephony-credential-preflight.ts +0 -10
  30. package/src/calls/telephony-tts-capability.ts +1 -17
  31. package/src/cli/commands/tts.ts +1 -2
  32. package/src/cli/index.help.ts +0 -2
  33. package/src/cli/program.ts +0 -2
  34. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +4 -42
  35. package/src/config/schemas/__tests__/stt.test.ts +10 -14
  36. package/src/config/schemas/llm.ts +3 -5
  37. package/src/config/schemas/services.ts +5 -6
  38. package/src/config/schemas/stt.ts +1 -3
  39. package/src/config/schemas/tts.ts +1 -30
  40. package/src/daemon/daemon-readiness.ts +4 -29
  41. package/src/daemon/lifecycle.ts +0 -31
  42. package/src/daemon/providers-setup.ts +0 -8
  43. package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
  44. package/src/heartbeat/heartbeat-service.ts +1 -66
  45. package/src/ipc/__tests__/db-readiness-ipc-gate.test.ts +1 -9
  46. package/src/live-voice/__tests__/live-voice-credential-preflight.test.ts +0 -44
  47. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +4 -6
  48. package/src/live-voice/__tests__/live-voice-stt.test.ts +1 -1
  49. package/src/live-voice/live-voice-credential-preflight.ts +9 -33
  50. package/src/live-voice/live-voice-session.ts +10 -6
  51. package/src/live-voice/live-voice-tts.ts +1 -8
  52. package/src/plugins/defaults/memory/worker.ts +3 -3
  53. package/src/prompts/persona-resolver.ts +0 -35
  54. package/src/providers/__tests__/search-provider-catalog.test.ts +5 -8
  55. package/src/providers/__tests__/voice-error-copy.test.ts +9 -10
  56. package/src/providers/search-provider-catalog.ts +7 -39
  57. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +3 -47
  58. package/src/providers/speech-to-text/__tests__/resolve.test.ts +6 -372
  59. package/src/providers/speech-to-text/provider-catalog.ts +2 -73
  60. package/src/providers/speech-to-text/resolve.ts +3 -120
  61. package/src/runtime/http-router-types.ts +1 -3
  62. package/src/runtime/http-server.ts +1 -1
  63. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +13 -11
  64. package/src/runtime/routes/__tests__/tts-routes.test.ts +2 -6
  65. package/src/runtime/routes/conversation-query-routes.ts +0 -96
  66. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +7 -7
  67. package/src/runtime/routes/index.ts +0 -4
  68. package/src/runtime/routes/live-voice-routes.test.ts +4 -11
  69. package/src/runtime/routes/live-voice-routes.ts +8 -11
  70. package/src/runtime/routes/secret-routes.ts +0 -6
  71. package/src/runtime/routes/tts-routes.ts +0 -38
  72. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1 -7
  73. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +0 -4
  74. package/src/schedule/schedule-store.ts +0 -7
  75. package/src/schedule/scheduler.ts +0 -5
  76. package/src/schedule/worker.ts +1 -1
  77. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +2 -10
  78. package/src/stt/daemon-batch-transcriber.ts +4 -38
  79. package/src/stt/types.ts +1 -9
  80. package/src/tools/network/__tests__/web-search.test.ts +3 -250
  81. package/src/tools/network/web-search.ts +4 -165
  82. package/src/tts/__tests__/provider-resolution.test.ts +0 -5
  83. package/src/tts/provider-catalog.ts +0 -4
  84. package/src/tts/types.ts +0 -1
  85. package/src/__tests__/cloud-speech-defaults.test.ts +0 -330
  86. package/src/__tests__/migration-cross-version-compatibility.test.ts +0 -1708
  87. package/src/__tests__/migration-export-http.test.ts +0 -557
  88. package/src/__tests__/migration-export-streaming.test.ts +0 -350
  89. package/src/__tests__/migration-export-to-gcs.test.ts +0 -706
  90. package/src/__tests__/migration-import-commit-http.test.ts +0 -1176
  91. package/src/__tests__/migration-import-from-gcs.test.ts +0 -534
  92. package/src/__tests__/migration-import-from-path.test.ts +0 -324
  93. package/src/__tests__/migration-import-from-url.test.ts +0 -588
  94. package/src/__tests__/migration-import-preflight-http.test.ts +0 -739
  95. package/src/__tests__/migration-jobs-status.test.ts +0 -139
  96. package/src/__tests__/migration-parity-persistence.test.ts +0 -1891
  97. package/src/__tests__/migration-transport.test.ts +0 -1089
  98. package/src/__tests__/migration-validate-http.test.ts +0 -634
  99. package/src/__tests__/migration-wizard.test.ts +0 -1395
  100. package/src/__tests__/rebind-secrets-screen.test.ts +0 -876
  101. package/src/__tests__/sanitize-config-for-transfer.test.ts +0 -259
  102. package/src/__tests__/transfer-progress-screen.test.ts +0 -1217
  103. package/src/__tests__/validation-results-screen.test.ts +0 -1130
  104. package/src/__tests__/vbundle-pax-and-symlink.test.ts +0 -147
  105. package/src/background-wake/background-wake-routes.test.ts +0 -906
  106. package/src/background-wake/cloud-client.test.ts +0 -328
  107. package/src/background-wake/cloud-client.ts +0 -196
  108. package/src/background-wake/next-wake.test.ts +0 -335
  109. package/src/background-wake/next-wake.ts +0 -188
  110. package/src/background-wake/publisher.ts +0 -93
  111. package/src/background-wake/wake-intent-hooks.test.ts +0 -234
  112. package/src/cli/commands/__tests__/backup.test.ts +0 -363
  113. package/src/cli/commands/backup.help.ts +0 -180
  114. package/src/cli/commands/backup.ts +0 -398
  115. package/src/cloud/cloud-speech.test.ts +0 -335
  116. package/src/cloud/cloud-speech.ts +0 -300
  117. package/src/config/cloud-speech-defaults.ts +0 -300
  118. package/src/config/sanitize-for-transfer.ts +0 -111
  119. package/src/live-voice/__tests__/live-voice-cloud-speech-fallback.test.ts +0 -250
  120. package/src/providers/speech-to-text/__tests__/cadet-cloud-realtime.test.ts +0 -457
  121. package/src/providers/speech-to-text/__tests__/cadet-cloud.test.ts +0 -271
  122. package/src/providers/speech-to-text/__tests__/cadet-speech-relay-connection.test.ts +0 -126
  123. package/src/providers/speech-to-text/cadet-cloud-flux-realtime.ts +0 -309
  124. package/src/providers/speech-to-text/cadet-cloud-realtime.ts +0 -315
  125. package/src/providers/speech-to-text/cadet-cloud.ts +0 -108
  126. package/src/providers/speech-to-text/cadet-speech-relay-connection.ts +0 -176
  127. package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +0 -254
  128. package/src/runtime/migrations/__tests__/job-registry.test.ts +0 -346
  129. package/src/runtime/migrations/__tests__/origin-mode.test.ts +0 -62
  130. package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +0 -184
  131. package/src/runtime/migrations/__tests__/staged-import-path.test.ts +0 -110
  132. package/src/runtime/migrations/__tests__/v1-test-helpers.ts +0 -112
  133. package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +0 -285
  134. package/src/runtime/migrations/__tests__/vbundle-builder-fd-leak.test.ts +0 -126
  135. package/src/runtime/migrations/__tests__/vbundle-builder-v1-shape.test.ts +0 -253
  136. package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +0 -208
  137. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +0 -485
  138. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +0 -349
  139. package/src/runtime/migrations/__tests__/vbundle-import-version-compat.test.ts +0 -196
  140. package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +0 -551
  141. package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +0 -428
  142. package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +0 -220
  143. package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +0 -1823
  144. package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +0 -598
  145. package/src/runtime/migrations/__tests__/vbundle-symlink-importer.test.ts +0 -463
  146. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -538
  147. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming.test.ts +0 -515
  148. package/src/runtime/migrations/__tests__/vbundle-symlink-tar.test.ts +0 -453
  149. package/src/runtime/migrations/__tests__/vbundle-symlink-walker.test.ts +0 -319
  150. package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +0 -240
  151. package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +0 -473
  152. package/src/runtime/migrations/gcs-signed-url.ts +0 -190
  153. package/src/runtime/migrations/job-registry.ts +0 -297
  154. package/src/runtime/migrations/migration-transport.ts +0 -612
  155. package/src/runtime/migrations/migration-wizard.ts +0 -747
  156. package/src/runtime/migrations/origin-mode.ts +0 -42
  157. package/src/runtime/migrations/rebind-secrets-screen.ts +0 -412
  158. package/src/runtime/migrations/staged-import-path.ts +0 -116
  159. package/src/runtime/migrations/transfer-progress-screen.ts +0 -323
  160. package/src/runtime/migrations/validation-results-screen.ts +0 -469
  161. package/src/runtime/migrations/vbundle-builder.ts +0 -1382
  162. package/src/runtime/migrations/vbundle-import-analyzer.ts +0 -405
  163. package/src/runtime/migrations/vbundle-import-policy.ts +0 -241
  164. package/src/runtime/migrations/vbundle-importer.ts +0 -826
  165. package/src/runtime/migrations/vbundle-metadata-merge.ts +0 -113
  166. package/src/runtime/migrations/vbundle-streaming-importer.ts +0 -2844
  167. package/src/runtime/migrations/vbundle-streaming-validator.ts +0 -426
  168. package/src/runtime/migrations/vbundle-tar-stream.ts +0 -244
  169. package/src/runtime/migrations/vbundle-validator.ts +0 -724
  170. package/src/runtime/routes/__tests__/migration-cadet-metadata-reconcile.test.ts +0 -227
  171. package/src/runtime/routes/__tests__/migration-export-secrets-redacted.test.ts +0 -54
  172. package/src/runtime/routes/__tests__/migration-import-allowed-hosts-env.test.ts +0 -70
  173. package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +0 -257
  174. package/src/runtime/routes/background-wake-routes.ts +0 -387
  175. package/src/runtime/routes/migration-routes.ts +0 -2584
  176. package/src/tools/network/__tests__/cloud-search-proxy.test.ts +0 -282
  177. package/src/tools/network/cloud-search-proxy.ts +0 -211
  178. package/src/tts/__tests__/cadet-provider.test.ts +0 -253
  179. package/src/tts/providers/cadet-provider.ts +0 -244
  180. package/src/types/tar-stream.d.ts +0 -66
@@ -1,2844 +0,0 @@
1
- /**
2
- * Streaming `.vbundle` importer.
3
- *
4
- * Buffer-based `commitImport` decompresses the whole archive into RAM and
5
- * re-walks the tar to write each file — fine for small bundles, OOMs on an
6
- * 8 GB bundle running on a 3 GB pod. This module orchestrates the streaming
7
- * primitives (`parseVBundleStream`, `readAndValidateManifest`,
8
- * `createHashVerifier`) to import a bundle with peak memory bounded by
9
- * "one tar entry size", not bundle size.
10
- *
11
- * Atomicity is provided by a temp-dir + double-rename pattern:
12
- *
13
- * 1. Entries land in `${workspaceDir}.import-<uuid>/` as they arrive, each
14
- * byte verified against the manifest's declared sha256/size before it
15
- * reaches disk.
16
- * 2. After every declared entry is accounted for, the live DB connection
17
- * is closed (`resetDb`) and the real workspace is swapped:
18
- * `rename(workspaceDir, backupDir)`
19
- * `rename(tempWorkspaceDir, workspaceDir)`
20
- * — atomic on POSIX. If the second rename fails we restore the backup.
21
- * 3. Post-commit side effects (credential import into CES, config/trust
22
- * cache invalidation) run after the swap. Failures here are non-fatal
23
- * — the workspace is already consistent.
24
- *
25
- * On any error before the rename pair, the temp workspace is removed and the
26
- * real workspace is left untouched.
27
- */
28
-
29
- import { createHash, randomUUID } from "node:crypto";
30
- import { createWriteStream, existsSync } from "node:fs";
31
- import {
32
- copyFile,
33
- cp,
34
- lstat,
35
- mkdir,
36
- readdir,
37
- readFile,
38
- readlink,
39
- rename,
40
- rm,
41
- stat,
42
- symlink,
43
- unlink,
44
- writeFile,
45
- } from "node:fs/promises";
46
- import { basename, dirname, join, resolve, sep } from "node:path";
47
- import { type Readable, Writable } from "node:stream";
48
- import { pipeline } from "node:stream/promises";
49
-
50
- import { z } from "zod";
51
-
52
- import { invalidateConfigCache } from "../../config/loader.js";
53
- import { sanitizeConfigForTransfer } from "../../config/sanitize-for-transfer.js";
54
- import { resetDb } from "../../persistence/db-connection.js";
55
- import { isGuardianPersonaCustomized } from "../../prompts/persona-resolver.js";
56
- import { getLogger } from "../../util/logger.js";
57
- import { APP_VERSION } from "../../version.js";
58
- import type { PathResolver } from "./vbundle-import-analyzer.js";
59
- import * as policy from "./vbundle-import-policy.js";
60
- import type {
61
- ImportCommitReport,
62
- ImportCommitResult,
63
- ImportedFileReport,
64
- ImportFileAction,
65
- } from "./vbundle-importer.js";
66
- import { mergeMetadataPreservingCadet } from "./vbundle-metadata-merge.js";
67
- import {
68
- createHashVerifier,
69
- readAndValidateManifest,
70
- StreamingValidationError,
71
- verifySymlinkEntry,
72
- } from "./vbundle-streaming-validator.js";
73
- import { parseVBundleStream } from "./vbundle-tar-stream.js";
74
- import {
75
- MANIFEST_MAX_ENTRIES,
76
- type ManifestType,
77
- } from "./vbundle-validator.js";
78
-
79
- const log = getLogger("vbundle-streaming-importer");
80
-
81
- // ---------------------------------------------------------------------------
82
- // Resource ceilings
83
- //
84
- // These cap the streaming importer's exposure to attacker-controlled bundle
85
- // inputs (e.g. a signed-URL migration from an untrusted source). Both caps
86
- // are exposed as optional `opts.maxBundleBytes` / `opts.maxBundleEntries`
87
- // parameters so tests can exercise the abort path with small fixtures —
88
- // production callers should omit the opts and rely on the defaults.
89
- // ---------------------------------------------------------------------------
90
-
91
- /**
92
- * Byte ceiling for the cumulative size of all file data streamed from the
93
- * bundle. 16 GiB gives comfortable headroom over the 8 GB product limit
94
- * while still bounding worst-case disk use for the temp workspace.
95
- */
96
- const DEFAULT_MAX_BUNDLE_BYTES = 16 * 1024 * 1024 * 1024;
97
-
98
- /**
99
- * Entry-count ceiling for the bundle, shared with manifest validation so
100
- * preflight and import agree. Also applied to tar-level entries so padding
101
- * extras cannot exceed what the manifest declared.
102
- */
103
- const DEFAULT_MAX_BUNDLE_ENTRIES = MANIFEST_MAX_ENTRIES;
104
-
105
- /**
106
- * Prefixes used for scratch dirs the streaming importer creates INSIDE the
107
- * workspace. Dot-prefixed to stay out of the way of real workspace content.
108
- * Phase 1 of `swapWorkspaceContents` skips the EXACT scratch basenames for
109
- * this run (via a `Set<string>` built from the backupDir/tempWorkspaceDir
110
- * basenames), so a user entry that happens to start with one of these
111
- * prefixes is still swept into the swap.
112
- *
113
- * Exported so tests asserting "no orphan temp/backup dirs" stay in sync with
114
- * the actual layout. Both dirs are created at `${workspaceDir}/<prefix><uuid>`
115
- * (i.e. INSIDE workspaceDir, not as a sibling).
116
- */
117
- export const IMPORT_TEMP_PREFIX = ".import-";
118
- export const IMPORT_BACKUP_PREFIX = ".pre-import-";
119
-
120
- // ---------------------------------------------------------------------------
121
- // Public API
122
- // ---------------------------------------------------------------------------
123
-
124
- export interface StreamProgressEvent {
125
- /** Archive path of the entry that just finished streaming. */
126
- archivePath: string;
127
- /** Total bytes written for that entry (equals manifest-declared size on success). */
128
- bytesWritten: number;
129
- /**
130
- * Zero-based index of the entry in the order it arrived in the tar. The
131
- * manifest itself is index 0; the first file entry is index 1.
132
- */
133
- entryIndex: number;
134
- }
135
-
136
- export interface StreamCommitArgs {
137
- /** Byte source for the `.vbundle`. Typically an HTTP response body. */
138
- source: Readable;
139
- /** Maps archive paths to their canonical disk locations. */
140
- pathResolver: PathResolver;
141
- /** Absolute path to the real workspace directory. */
142
- workspaceDir: string;
143
- /** Optional progress callback invoked after each file entry finishes. */
144
- onProgress?: (evt: StreamProgressEvent) => void;
145
- /**
146
- * Optional callback for importing credentials into CES after the atomic
147
- * swap succeeds. Failures are treated as non-fatal warnings. When omitted,
148
- * credentials discovered in the bundle are ignored — the caller
149
- * (`migration-routes.ts`) is responsible for wiring this.
150
- */
151
- importCredentials?: (
152
- credentials: Array<{ account: string; value: string }>,
153
- ) => Promise<void>;
154
- /**
155
- * Test-only override for the bundle-size ceiling (bytes). Production
156
- * callers should omit this and rely on the 16 GiB default.
157
- */
158
- maxBundleBytes?: number;
159
- /**
160
- * Test-only override for the entry-count ceiling. Production callers
161
- * should omit this and rely on the `MANIFEST_MAX_ENTRIES` default.
162
- */
163
- maxBundleEntries?: number;
164
- }
165
-
166
- /**
167
- * Stream a `.vbundle` archive from `source` and commit it to disk atomically.
168
- *
169
- * Returns an `ImportCommitResult` matching the shape produced by the
170
- * buffer-based `commitImport`, so callers can treat the two paths
171
- * interchangeably.
172
- */
173
- export async function streamCommitImport(
174
- args: StreamCommitArgs,
175
- ): Promise<ImportCommitResult> {
176
- const {
177
- source,
178
- pathResolver,
179
- workspaceDir,
180
- onProgress,
181
- importCredentials,
182
- maxBundleBytes,
183
- maxBundleEntries,
184
- } = args;
185
-
186
- const bundleByteCap = maxBundleBytes ?? DEFAULT_MAX_BUNDLE_BYTES;
187
- const bundleEntryCap = maxBundleEntries ?? DEFAULT_MAX_BUNDLE_ENTRIES;
188
-
189
- const realWorkspaceDir = resolve(workspaceDir);
190
-
191
- // Replay recovery from any prior interrupted import BEFORE we stage
192
- // new data. If the previous import died mid-swap, the marker / temp /
193
- // backup still sit in the workspace and recoverInterruptedImport rolls
194
- // them back. If that rollback is INCOMPLETE (per-entry restore failed
195
- // and we had to preserve the marker for retry), we must REFUSE to
196
- // start a new import — this function is about to rewrite the marker
197
- // at the same path, and a fresh write would orphan the unresolved
198
- // backup/temp pointers, making the interrupted state unrecoverable.
199
- //
200
- // In that case, return write_failed so the caller retries later; an
201
- // operator can investigate the leftover `.pre-import-*` / `.import-*`
202
- // dirs in the workspace.
203
- let recoveryResult: RecoveryResult;
204
- try {
205
- recoveryResult = await recoverInterruptedImport(realWorkspaceDir);
206
- } catch (err) {
207
- log.error(
208
- { err, realWorkspaceDir },
209
- "recoverInterruptedImport threw before streaming import",
210
- );
211
- return {
212
- ok: false,
213
- reason: "write_failed",
214
- message: `Pre-import recovery failed: ${errMessage(err)}`,
215
- };
216
- }
217
- if (!recoveryResult.ok) {
218
- log.error(
219
- {
220
- realWorkspaceDir,
221
- failedCount: recoveryResult.failedCount,
222
- },
223
- "Previous import rollback is still unresolved; refusing to start a new import",
224
- );
225
- return {
226
- ok: false,
227
- reason: "write_failed",
228
- message:
229
- `Previous import rollback is still unresolved (${recoveryResult.failedCount} entries failed to restore). ` +
230
- "Leftover backup/temp dirs are preserved in the workspace; manual intervention may be required before the next import.",
231
- };
232
- }
233
-
234
- // Put scratch dirs (temp staging tree, backup dir) INSIDE the workspace
235
- // mount so every move during the content-level swap stays on the same
236
- // filesystem. If they lived as siblings (on the container overlay),
237
- // every rename in swapWorkspaceContents would cross filesystems and
238
- // require a full cp+rm of the entire workspace. That defeats the
239
- // zero-disk fast path and risks ENOSPC on the overlay for large
240
- // teleports. Dot-prefixed names keep them out of the way of normal
241
- // content; phase 1 of swapWorkspaceContents filters them out by exact
242
- // basename so user entries that happen to start with these prefixes
243
- // are still swept through the swap.
244
- const tempWorkspaceDir = join(
245
- realWorkspaceDir,
246
- `${IMPORT_TEMP_PREFIX}${randomUUID()}`,
247
- );
248
-
249
- let manifest: ManifestType | null = null;
250
- const importedFiles: ImportedFileReport[] = [];
251
- const warnings: string[] = [];
252
- const seen = new Set<string>();
253
- // Credential bodies are small (API keys / tokens) — safe to buffer in
254
- // memory. They intentionally never touch disk: DefaultPathResolver returns
255
- // null for `credentials/*`, and CES is the only consumer.
256
- const bufferedCredentials: Array<{ account: string; value: string }> = [];
257
- // Track whether the bundle contains at least one `workspace/*` entry that
258
- // resolves to a real disk path. The atomic swap path (which wipes anything
259
- // outside WORKSPACE_PRESERVE_PATHS) is only safe to take when this is
260
- // true — it matches commitImport's `hasWorkspaceEntries` gate. Legacy
261
- // bundles (e.g. `data/db/*`, `config/*`, `prompts/*`, `skills/*` without a
262
- // workspace/ prefix) fall through to the in-place write path below.
263
- let hasWorkspaceNamespacedEntry = false;
264
- // Accumulates the disk paths of files we staged into the temp workspace
265
- // from legacy-format archive entries. If the bundle turns out to contain
266
- // NO workspace/ entries we promote each of these into the live workspace
267
- // with backup-before-overwrite semantics, matching commitImport's legacy
268
- // handling. Each tuple carries (tempPath, livePath, archivePath, index).
269
- const legacyStaged: Array<{
270
- tempPath: string;
271
- livePath: string;
272
- archivePath: string;
273
- importedFileIndex: number;
274
- }> = [];
275
- // Cumulative manifest-declared byte total, accumulated BEFORE each entry
276
- // is read/written. Checked against `bundleByteCap` pre-write so an
277
- // oversized entry never lands on disk. We count manifest-declared
278
- // `expectedEntry.size` (the raw archive bytes) rather than on-disk size
279
- // so a sanitized config still counts against the cap as originally
280
- // declared.
281
- let totalBytesStreamed = 0;
282
- // Number of file/directory entries processed (not counting the manifest).
283
- // Compared against `bundleEntryCap`.
284
- let entryCount = 0;
285
-
286
- // The temp workspace dir is created lazily inside the parse loop AFTER
287
- // the manifest's version gate passes (see the `entryIndex === 0` block
288
- // below). Creating it up front would materialize `${workspaceDir}` (and
289
- // the `.import-<uuid>` subdir) on a fresh filesystem before we knew the
290
- // bundle was compatible — violating the plan invariant that importers
291
- // gate on runtime-version compat BEFORE any state mutation.
292
- //
293
- // `cleanupTempDir` is safe whether or not the dir was ever created:
294
- // `rm(..., { recursive: true, force: true })` is a no-op on a missing
295
- // path.
296
- const cleanupTempDir = async (): Promise<void> => {
297
- try {
298
- await rm(tempWorkspaceDir, { recursive: true, force: true });
299
- } catch (err) {
300
- log.warn(
301
- { err, tempWorkspaceDir },
302
- "Failed to clean up temp workspace dir after import failure",
303
- );
304
- }
305
- };
306
-
307
- // Iterate the tar stream. Any error from gzip/tar/source bubbles out of
308
- // the generator and lands in the catch block below.
309
- let entryIndex = 0;
310
- try {
311
- const entries = parseVBundleStream(source);
312
- let expected: Map<
313
- string,
314
- { sha256: string; size: number; linkTarget: string | null }
315
- > | null = null;
316
-
317
- for await (const entry of entries) {
318
- if (entryIndex === 0) {
319
- // First entry MUST be manifest.json — readAndValidateManifest
320
- // enforces that and throws StreamingValidationError otherwise.
321
- const manifestResult = await readAndValidateManifest(entry, {
322
- maxEntries: bundleEntryCap,
323
- });
324
- manifest = manifestResult.manifest;
325
- expected = manifestResult.expected;
326
-
327
- // Defense-in-depth: refuse to populate the temp tree when the
328
- // bundle's compat range excludes APP_VERSION. The version gate
329
- // runs BEFORE we materialize `${workspaceDir}/.import-<uuid>`
330
- // (the mkdir below is sequenced after this check), so on a fresh
331
- // filesystem an incompatible bundle leaves zero filesystem trace.
332
- // Throwing inside the generator's try block still triggers
333
- // cleanupTempDir() in the catch — a safe no-op on a missing path
334
- // — and mapThrownToResult translates VersionIncompatibleError into
335
- // the version_incompatible result shape. Catches legacy bundles
336
- // whose ExportJob row predates the cloud compat-column rollout
337
- // (compat columns NULL → platform gate skipped) and any future
338
- // drift between the cloud gate and the manifest.
339
- const compatResult = policy.evaluateRuntimeCompatibility(
340
- manifest.compatibility,
341
- APP_VERSION,
342
- );
343
- if (!compatResult.ok) {
344
- throw new VersionIncompatibleError(
345
- compatResult.bundle_compat,
346
- compatResult.runtime_version,
347
- );
348
- }
349
-
350
- // Only NOW, after the manifest is parsed (including its entry-count
351
- // ceiling) and the version gate passes, do we materialize the
352
- // temp staging dir on disk. Doing this lazily preserves the plan
353
- // invariant that importers gate on runtime-version compat BEFORE
354
- // any state mutation. If this throws, the outer catch runs
355
- // cleanupTempDir (a safe no-op on a missing path) and
356
- // mapThrownToResult translates the WriteFailedError into the
357
- // write_failed shape of ImportCommitResult.
358
- try {
359
- await mkdir(tempWorkspaceDir, { recursive: true });
360
- } catch (err) {
361
- throw wrapWriteError(
362
- `Failed to create temp workspace dir "${tempWorkspaceDir}"`,
363
- err,
364
- );
365
- }
366
-
367
- entryIndex += 1;
368
- continue;
369
- }
370
-
371
- // After the manifest we must have `expected` populated.
372
- if (!manifest || !expected) {
373
- throw new StreamingValidationError(
374
- "manifest_not_first",
375
- "Manifest processing did not complete before subsequent entries",
376
- );
377
- }
378
-
379
- // Entry-count ceiling also applies to tar-level entries that arrive
380
- // in the stream (pax headers, directories, extras). A bundle whose
381
- // manifest stayed under the cap but whose tar carries padding-style
382
- // extras is still bounded.
383
- entryCount += 1;
384
- if (entryCount > bundleEntryCap) {
385
- entry.body.destroy();
386
- throw new StreamingValidationError(
387
- "bundle_too_many_entries",
388
- `bundle contains more than ${bundleEntryCap} entries`,
389
- );
390
- }
391
-
392
- const archivePath = entry.header.name;
393
-
394
- // Non-file entries are either directory markers (empty body) or
395
- // pax-header / other metadata payloads we don't consume. Apply the
396
- // bundle byte cap to their tar-header size too — an attacker could
397
- // otherwise keep `manifest.contents` small while stuffing huge pax/other
398
- // entry bodies, draining the importer for free. Directory bodies are
399
- // reliably zero-sized; pax headers are measured in bytes, so this
400
- // check is effectively free in the happy path.
401
- if (entry.header.type !== "file") {
402
- const nonFileSize = entry.header.size ?? 0;
403
- if (totalBytesStreamed + nonFileSize > bundleByteCap) {
404
- entry.body.destroy();
405
- throw new StreamingValidationError(
406
- "bundle_too_large",
407
- `bundle exceeds ${bundleByteCap}-byte ceiling (non-file entry "${archivePath}" size ${nonFileSize})`,
408
- archivePath,
409
- );
410
- }
411
- totalBytesStreamed += nonFileSize;
412
- }
413
-
414
- if (entry.header.type === "directory") {
415
- // Best-effort: create the directory inside the temp workspace if it
416
- // resolves inside `workspaceDir`. Drain the empty body either way.
417
- entry.body.resume();
418
- const dirResolved = resolveInsideTempWorkspace(
419
- archivePath,
420
- pathResolver,
421
- realWorkspaceDir,
422
- tempWorkspaceDir,
423
- );
424
- if (dirResolved) {
425
- try {
426
- await mkdir(dirResolved, { recursive: true });
427
- } catch (err) {
428
- throw wrapWriteError(
429
- `Failed to create directory "${dirResolved}"`,
430
- err,
431
- );
432
- }
433
- }
434
- entryIndex += 1;
435
- continue;
436
- }
437
-
438
- if (entry.header.type !== "file" && entry.header.type !== "symlink") {
439
- // pax-header / other — drain and skip. Non-file payloads are
440
- // metadata for the tar extractor itself, not user data.
441
- entry.body.resume();
442
- entryIndex += 1;
443
- continue;
444
- }
445
-
446
- const expectedEntry = expected.get(archivePath);
447
- if (!expectedEntry) {
448
- // Bundle contains a file the manifest didn't declare. Destroy the
449
- // body so the extractor aborts promptly.
450
- entry.body.destroy();
451
- throw new StreamingValidationError(
452
- "manifest_mismatch",
453
- `Archive entry "${archivePath}" is not declared in the manifest`,
454
- archivePath,
455
- );
456
- }
457
-
458
- // Symlink branch: typeflag-2 entry, OR a regular-file tar entry whose
459
- // manifest declared `link_target`. `verifySymlinkEntry` cross-validates
460
- // both directions — tar symlink without manifest link_target,
461
- // tar regular file with manifest link_target, linkname/manifest
462
- // disagreement, sha mismatch, traversal, absolute target. It also
463
- // drains the body so the tar extractor advances.
464
- if (
465
- entry.header.type === "symlink" ||
466
- expectedEntry.linkTarget !== null
467
- ) {
468
- verifySymlinkEntry({ entry, expectedEntry });
469
-
470
- // Defense-in-depth: even though verifySymlinkEntry rejected absolute
471
- // / `..` traversal, re-check from the IMPORTER perspective using the
472
- // resolved disk path (which maps archive paths through the resolver,
473
- // e.g. legacy `prompts/USER.md` -> `users/<slug>.md`).
474
- const linkTargetStr = expectedEntry.linkTarget as string;
475
- const diskPath = pathResolver.resolve(archivePath);
476
- if (!diskPath) {
477
- importedFiles.push({
478
- path: archivePath,
479
- disk_path: "",
480
- action: "skipped",
481
- size: 0,
482
- sha256: expectedEntry.sha256,
483
- backup_path: null,
484
- });
485
- warnings.push(
486
- `Skipped "${archivePath}": no known disk target for this archive path`,
487
- );
488
- seen.add(archivePath);
489
- onProgress?.({ archivePath, bytesWritten: 0, entryIndex });
490
- entryIndex += 1;
491
- continue;
492
- }
493
-
494
- const wsResolved = resolve(realWorkspaceDir);
495
- const targetResolved = resolve(dirname(diskPath), linkTargetStr);
496
- if (
497
- linkTargetStr.startsWith("/") ||
498
- (targetResolved !== wsResolved &&
499
- !targetResolved.startsWith(wsResolved + sep))
500
- ) {
501
- importedFiles.push({
502
- path: archivePath,
503
- disk_path: diskPath,
504
- action: "skipped",
505
- size: 0,
506
- sha256: expectedEntry.sha256,
507
- backup_path: null,
508
- });
509
- warnings.push(
510
- `Skipped "${archivePath}": symlink target "${linkTargetStr}" escapes workspace`,
511
- );
512
- seen.add(archivePath);
513
- onProgress?.({ archivePath, bytesWritten: 0, entryIndex });
514
- entryIndex += 1;
515
- continue;
516
- }
517
-
518
- // Legacy guardian persona protection — match commitImport's
519
- // behavior. If the bundle ships `prompts/USER.md` as a symlink and
520
- // the destination guardian persona is already user-customized,
521
- // skip rather than clobber.
522
- if (
523
- policy.isLegacyPersonaArchivePath(archivePath) &&
524
- isGuardianPersonaCustomized(diskPath)
525
- ) {
526
- log.warn(
527
- { archivePath, diskPath },
528
- "Skipping legacy prompts/USER.md symlink import: guardian persona is already customized",
529
- );
530
- importedFiles.push({
531
- path: archivePath,
532
- disk_path: diskPath,
533
- action: "skipped",
534
- size: 0,
535
- sha256: expectedEntry.sha256,
536
- backup_path: null,
537
- });
538
- warnings.push(
539
- `Skipped "${archivePath}": guardian persona at "${diskPath}" is already customized`,
540
- );
541
- seen.add(archivePath);
542
- onProgress?.({ archivePath, bytesWritten: 0, entryIndex });
543
- entryIndex += 1;
544
- continue;
545
- }
546
-
547
- // Rebase onto temp workspace so the swap moves the symlink into the
548
- // live workspace atomically.
549
- const tempDiskPath = rebaseOntoTempWorkspace(
550
- diskPath,
551
- realWorkspaceDir,
552
- tempWorkspaceDir,
553
- );
554
- if (!tempDiskPath) {
555
- importedFiles.push({
556
- path: archivePath,
557
- disk_path: diskPath,
558
- action: "skipped",
559
- size: 0,
560
- sha256: expectedEntry.sha256,
561
- backup_path: null,
562
- });
563
- warnings.push(
564
- `Skipped "${archivePath}": disk target "${diskPath}" falls outside the workspace directory`,
565
- );
566
- seen.add(archivePath);
567
- onProgress?.({ archivePath, bytesWritten: 0, entryIndex });
568
- entryIndex += 1;
569
- continue;
570
- }
571
-
572
- try {
573
- await mkdir(dirname(tempDiskPath), { recursive: true });
574
- } catch (err) {
575
- throw wrapWriteError(
576
- `Failed to create parent directory for "${tempDiskPath}"`,
577
- err,
578
- );
579
- }
580
-
581
- try {
582
- await symlink(linkTargetStr, tempDiskPath);
583
- } catch (err) {
584
- throw wrapWriteError(
585
- `Failed to create symlink "${tempDiskPath}" -> "${linkTargetStr}"`,
586
- err,
587
- );
588
- }
589
-
590
- const isWorkspaceNamespaced = archivePath.startsWith("workspace/");
591
- const importedFileIndex = importedFiles.length;
592
- importedFiles.push({
593
- path: archivePath,
594
- disk_path: diskPath,
595
- action: "created",
596
- size: 0,
597
- sha256: expectedEntry.sha256,
598
- backup_path: null,
599
- });
600
- if (isWorkspaceNamespaced) {
601
- hasWorkspaceNamespacedEntry = true;
602
- } else {
603
- legacyStaged.push({
604
- tempPath: tempDiskPath,
605
- livePath: diskPath,
606
- archivePath,
607
- importedFileIndex,
608
- });
609
- }
610
- seen.add(archivePath);
611
- onProgress?.({ archivePath, bytesWritten: 0, entryIndex });
612
- entryIndex += 1;
613
- continue;
614
- }
615
-
616
- // Reject tar entries whose declared size disagrees with the manifest.
617
- // The bundle-size ceiling below trusts `expectedEntry.size`; if a
618
- // crafted bundle declared a tiny size in `manifest.json` but carried a
619
- // huge body in the tar header, the cap would pass and the oversized
620
- // payload would still stream to disk. `createHashVerifier` already
621
- // fails on size mismatch at stream end, but by then the bytes have
622
- // already been written. Fail fast here so no oversized payload lands
623
- // on disk.
624
- if (entry.header.size !== expectedEntry.size) {
625
- entry.body.destroy();
626
- throw new StreamingValidationError(
627
- "entry_size",
628
- `Archive entry "${archivePath}" has tar-header size ${entry.header.size} but manifest declares ${expectedEntry.size}`,
629
- archivePath,
630
- );
631
- }
632
-
633
- // Enforce the bundle-size ceiling BEFORE writing/consuming the entry.
634
- // Checking post-write would still let a single oversized file land on
635
- // disk before we reject, defeating the cap as a resource guard. We
636
- // check both the manifest-declared size (what we just verified the
637
- // tar agrees with) AND the tar-header size directly, using whichever
638
- // is larger, so a future header/manifest desync can't slip through.
639
- const declaredSize = Math.max(entry.header.size, expectedEntry.size);
640
- if (totalBytesStreamed + declaredSize > bundleByteCap) {
641
- entry.body.destroy();
642
- throw new StreamingValidationError(
643
- "bundle_too_large",
644
- `bundle exceeds ${bundleByteCap}-byte ceiling`,
645
- archivePath,
646
- );
647
- }
648
- totalBytesStreamed += declaredSize;
649
-
650
- if (archivePath.startsWith("credentials/")) {
651
- // Credentials are hash-verified against the manifest but collected
652
- // in memory rather than written to disk. DefaultPathResolver
653
- // deliberately returns null for these paths.
654
- const buffered = await collectHashVerified(entry.body, {
655
- sha256: expectedEntry.sha256,
656
- size: expectedEntry.size,
657
- archivePath,
658
- });
659
- const account = archivePath.slice("credentials/".length);
660
- if (account) {
661
- bufferedCredentials.push({
662
- account,
663
- value: new TextDecoder().decode(buffered),
664
- });
665
- }
666
- seen.add(archivePath);
667
- onProgress?.({
668
- archivePath,
669
- bytesWritten: expectedEntry.size,
670
- entryIndex,
671
- });
672
- entryIndex += 1;
673
- continue;
674
- }
675
-
676
- const diskPath = pathResolver.resolve(archivePath);
677
- if (!diskPath) {
678
- // Unknown destination. Consume bytes through the verifier anyway so
679
- // we still catch manifest/content mismatches, but don't write.
680
- // Tracking this in the report matches the buffer-based importer's
681
- // "skipped" semantics.
682
- await drainThroughVerifier(entry.body, {
683
- sha256: expectedEntry.sha256,
684
- size: expectedEntry.size,
685
- archivePath,
686
- });
687
- importedFiles.push({
688
- path: archivePath,
689
- disk_path: "",
690
- action: "skipped",
691
- size: expectedEntry.size,
692
- sha256: expectedEntry.sha256,
693
- backup_path: null,
694
- });
695
- // Retired-feature paths (see `policy.RETIRED_ARCHIVE_PATHS`) are
696
- // expected in legacy bundles and skip silently so the import report
697
- // matches preflight; anything else unresolvable earns a warning.
698
- if (
699
- !policy.isRetiredArchivePath(archivePath) &&
700
- !policy.isGatewayArchivePath(archivePath)
701
- ) {
702
- warnings.push(
703
- `Skipped "${archivePath}": no known disk target for this archive path`,
704
- );
705
- }
706
- seen.add(archivePath);
707
- onProgress?.({
708
- archivePath,
709
- bytesWritten: expectedEntry.size,
710
- entryIndex,
711
- });
712
- entryIndex += 1;
713
- continue;
714
- }
715
-
716
- // Legacy guardian persona (prompts/USER.md) is translated to the
717
- // current guardian's users/<slug>.md by DefaultPathResolver. If
718
- // that target already holds user-authored content, skip rather
719
- // than clobber — the user has curated their persona since the
720
- // bundle was exported. We check against the LIVE workspace path
721
- // (diskPath) because the swap hasn't happened yet.
722
- if (
723
- policy.isLegacyPersonaArchivePath(archivePath) &&
724
- isGuardianPersonaCustomized(diskPath)
725
- ) {
726
- log.warn(
727
- { archivePath, diskPath },
728
- "Skipping legacy prompts/USER.md import: guardian persona is already customized",
729
- );
730
- await drainThroughVerifier(entry.body, {
731
- sha256: expectedEntry.sha256,
732
- size: expectedEntry.size,
733
- archivePath,
734
- });
735
- importedFiles.push({
736
- path: archivePath,
737
- disk_path: diskPath,
738
- action: "skipped",
739
- size: expectedEntry.size,
740
- sha256: expectedEntry.sha256,
741
- backup_path: null,
742
- });
743
- warnings.push(
744
- `Skipped "${archivePath}": guardian persona at "${diskPath}" is already customized`,
745
- );
746
- seen.add(archivePath);
747
- onProgress?.({
748
- archivePath,
749
- bytesWritten: expectedEntry.size,
750
- entryIndex,
751
- });
752
- entryIndex += 1;
753
- continue;
754
- }
755
-
756
- // Rebase the resolved path onto the temp workspace.
757
- const tempDiskPath = rebaseOntoTempWorkspace(
758
- diskPath,
759
- realWorkspaceDir,
760
- tempWorkspaceDir,
761
- );
762
- if (!tempDiskPath) {
763
- // Resolved outside the workspace directory. Not supported for the
764
- // streaming atomic-swap path — write through the verifier but flag
765
- // as skipped.
766
- await drainThroughVerifier(entry.body, {
767
- sha256: expectedEntry.sha256,
768
- size: expectedEntry.size,
769
- archivePath,
770
- });
771
- importedFiles.push({
772
- path: archivePath,
773
- disk_path: diskPath,
774
- action: "skipped",
775
- size: expectedEntry.size,
776
- sha256: expectedEntry.sha256,
777
- backup_path: null,
778
- });
779
- warnings.push(
780
- `Skipped "${archivePath}": disk target "${diskPath}" falls outside the workspace directory`,
781
- );
782
- seen.add(archivePath);
783
- onProgress?.({
784
- archivePath,
785
- bytesWritten: expectedEntry.size,
786
- entryIndex,
787
- });
788
- entryIndex += 1;
789
- continue;
790
- }
791
-
792
- try {
793
- await mkdir(dirname(tempDiskPath), { recursive: true });
794
- } catch (err) {
795
- throw wrapWriteError(
796
- `Failed to create parent directory for "${tempDiskPath}"`,
797
- err,
798
- );
799
- }
800
-
801
- // Classify the entry as `workspace/*` (namespaced) vs legacy format.
802
- // Namespaced entries flip the swap-gate flag; legacy entries are
803
- // staged for an in-place promote after the stream completes.
804
- const isWorkspaceNamespaced =
805
- policy.isWorkspaceNamespacedArchivePath(archivePath);
806
-
807
- // Config files need sanitization before writing to strip
808
- // environment-specific fields (defense-in-depth; matches commitImport).
809
- // Configs are small (KB-scale) so buffering them is fine. Hash
810
- // verification still runs on the RAW bytes — the manifest declares the
811
- // sha/size of the archive content, not the sanitized output.
812
- if (policy.isConfigArchivePath(archivePath)) {
813
- const rawBytes = await collectHashVerified(entry.body, {
814
- sha256: expectedEntry.sha256,
815
- size: expectedEntry.size,
816
- archivePath,
817
- });
818
- const sanitized = sanitizeConfigForTransfer(
819
- new TextDecoder().decode(rawBytes),
820
- );
821
- const sanitizedBytes = new TextEncoder().encode(sanitized);
822
- try {
823
- await writeFile(tempDiskPath, sanitizedBytes, { mode: 0o600 });
824
- } catch (err) {
825
- throw wrapWriteError(`Failed to write "${tempDiskPath}"`, err);
826
- }
827
- // commitImport reports the sha256 of the bytes actually written to
828
- // disk (which differs from the manifest-declared sha once
829
- // sanitization strips fields). Mirror that here so downstream
830
- // integrity re-checks against the on-disk file succeed.
831
- const onDiskSha = sha256Hex(sanitizedBytes);
832
- const importedFileIndex = importedFiles.length;
833
- importedFiles.push({
834
- path: archivePath,
835
- disk_path: diskPath,
836
- action: "created",
837
- // Report the sanitized on-disk size, not the archive's raw size —
838
- // matches what commitImport reports.
839
- size: sanitizedBytes.length,
840
- sha256: onDiskSha,
841
- backup_path: null,
842
- });
843
- if (isWorkspaceNamespaced) {
844
- hasWorkspaceNamespacedEntry = true;
845
- } else {
846
- legacyStaged.push({
847
- tempPath: tempDiskPath,
848
- livePath: diskPath,
849
- archivePath,
850
- importedFileIndex,
851
- });
852
- }
853
- seen.add(archivePath);
854
- onProgress?.({
855
- archivePath,
856
- bytesWritten: expectedEntry.size,
857
- entryIndex,
858
- });
859
- entryIndex += 1;
860
- continue;
861
- }
862
-
863
- const verifier = createHashVerifier({
864
- sha256: expectedEntry.sha256,
865
- size: expectedEntry.size,
866
- archivePath,
867
- });
868
- const writeStream = createWriteStream(tempDiskPath, { mode: 0o600 });
869
- try {
870
- await pipeline(entry.body, verifier, writeStream);
871
- } catch (err) {
872
- // Disambiguate between hash/size validation failures and raw disk
873
- // write errors so the caller sees the right reason code.
874
- if (err instanceof StreamingValidationError) {
875
- throw err;
876
- }
877
- throw wrapWriteError(`Failed to write "${tempDiskPath}"`, err);
878
- }
879
-
880
- // Action is "created" for the in-temp-tree record. Whether the real
881
- // workspace sees this as create vs overwrite is resolved later: the
882
- // atomic-swap path wipes and replaces wholesale, while the legacy
883
- // in-place promote checks against the live file and flips the action
884
- // to "overwritten" with a backup.
885
- const action: ImportFileAction = "created";
886
- const importedFileIndex = importedFiles.length;
887
- importedFiles.push({
888
- path: archivePath,
889
- disk_path: diskPath,
890
- action,
891
- size: expectedEntry.size,
892
- sha256: expectedEntry.sha256,
893
- backup_path: null,
894
- });
895
- if (isWorkspaceNamespaced) {
896
- hasWorkspaceNamespacedEntry = true;
897
- } else {
898
- legacyStaged.push({
899
- tempPath: tempDiskPath,
900
- livePath: diskPath,
901
- archivePath,
902
- importedFileIndex,
903
- });
904
- }
905
- seen.add(archivePath);
906
- onProgress?.({
907
- archivePath,
908
- bytesWritten: expectedEntry.size,
909
- entryIndex,
910
- });
911
- entryIndex += 1;
912
- }
913
-
914
- // Manifest must have been processed.
915
- if (!manifest || !expected) {
916
- throw new StreamingValidationError(
917
- "manifest_not_first",
918
- "Archive contained no entries",
919
- );
920
- }
921
-
922
- // Every declared manifest path must have been seen in the tar stream.
923
- const missing: string[] = [];
924
- for (const path of expected.keys()) {
925
- if (!seen.has(path)) {
926
- missing.push(path);
927
- }
928
- }
929
- if (missing.length > 0) {
930
- throw new StreamingValidationError(
931
- "missing_entry",
932
- `Bundle is missing ${missing.length} declared entr${
933
- missing.length === 1 ? "y" : "ies"
934
- }: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ", …" : ""}`,
935
- missing[0],
936
- );
937
- }
938
- } catch (err) {
939
- await cleanupTempDir();
940
- return mapThrownToResult(err);
941
- }
942
-
943
- // -------------------------------------------------------------------------
944
- // Commit strategy selection
945
- //
946
- // commitImport's in-place path only clears the workspace when the bundle
947
- // carries at least one `workspace/*` entry that resolves to a real disk
948
- // path — legacy-format bundles (`data/db/*`, `config/*`, `prompts/*`,
949
- // `skills/*`, `hooks/*` without a workspace/ prefix) write individual
950
- // files in place without wiping siblings. The streaming importer's
951
- // atomic-swap path is equivalent to the selective-clear-and-write path;
952
- // it must therefore only fire when `hasWorkspaceNamespacedEntry` is
953
- // true. For legacy-only bundles we promote staged temp files into the
954
- // live workspace one by one with backup-before-overwrite semantics.
955
- // -------------------------------------------------------------------------
956
-
957
- // Empty result: no writable entries, no staged legacy files. Skip both
958
- // commit paths — nothing can alter the live workspace. This matches
959
- // commitImport's no-op behavior for all-credential or all-skipped
960
- // bundles.
961
- if (!hasWorkspaceNamespacedEntry && legacyStaged.length === 0) {
962
- await cleanupTempDir();
963
-
964
- // Post-commit side effects still run for things like credential import.
965
- if (importCredentials && bufferedCredentials.length > 0) {
966
- try {
967
- await importCredentials(bufferedCredentials);
968
- } catch (err) {
969
- log.warn(
970
- { err, count: bufferedCredentials.length },
971
- "Post-commit credential import failed",
972
- );
973
- warnings.push(`Credential import failed: ${errMessage(err)}`);
974
- }
975
- }
976
-
977
- const report = buildReport(manifest, importedFiles, warnings);
978
- return { ok: true, report };
979
- }
980
-
981
- // Legacy-only bundle: we have files staged under the temp workspace but
982
- // no `workspace/*` entries telling us the caller wants to replace the
983
- // entire workspace. Promote each staged file into the live workspace in
984
- // place, matching commitImport's legacy branch (backup-before-overwrite,
985
- // parent-dir mkdir, no workspace-wide clear). The temp workspace is
986
- // removed when done — it only served as a landing zone for the verified
987
- // hash stream.
988
- if (!hasWorkspaceNamespacedEntry) {
989
- // Close the live SQLite connection before promoting staged files. A
990
- // legacy bundle may carry `data/db/assistant.db`, and replacing the file
991
- // with an open connection leaves the daemon pinned to the old inode —
992
- // subsequent reads/writes would go against stale pre-import data until
993
- // the process reset the connection. The singleton lazily reopens on next
994
- // use, so closing here is safe even if no DB entry is in the bundle.
995
- try {
996
- resetDb();
997
- } catch (err) {
998
- log.warn(
999
- { err },
1000
- "resetDb threw before legacy-format import promotion; continuing",
1001
- );
1002
- }
1003
-
1004
- try {
1005
- await promoteLegacyStagedFiles(legacyStaged, importedFiles);
1006
- } catch (err) {
1007
- // Legacy promotion mutates live files one at a time, so a mid-loop
1008
- // failure leaves an observable partial import: every entry in
1009
- // `importedFiles` whose `action` has flipped from "created" (the
1010
- // temp-staged state) to "overwritten" or that now carries a
1011
- // `backup_path` has landed on live disk. Report that back so callers
1012
- // can tell what changed, matching commitImport's partial_report
1013
- // contract for its in-place path.
1014
- const partialReport = buildReport(manifest, importedFiles, warnings);
1015
- await cleanupTempDir();
1016
- return {
1017
- ok: false,
1018
- reason: "write_failed",
1019
- message: `Failed to promote legacy-format import into workspace: ${errMessage(err)}`,
1020
- partial_report: partialReport,
1021
- };
1022
- }
1023
-
1024
- await cleanupTempDir();
1025
-
1026
- // Post-commit side effects. Config/trust caches can still be stale
1027
- // from a legacy config/settings.json write, and credentials still
1028
- // need to flow through CES.
1029
- if (importCredentials && bufferedCredentials.length > 0) {
1030
- try {
1031
- await importCredentials(bufferedCredentials);
1032
- } catch (err) {
1033
- log.warn(
1034
- { err, count: bufferedCredentials.length },
1035
- "Post-commit credential import failed",
1036
- );
1037
- warnings.push(`Credential import failed: ${errMessage(err)}`);
1038
- }
1039
- }
1040
-
1041
- try {
1042
- invalidateConfigCache();
1043
- } catch (err) {
1044
- log.warn({ err }, "invalidateConfigCache threw after legacy import");
1045
- }
1046
-
1047
- const report = buildReport(manifest, importedFiles, warnings);
1048
- return { ok: true, report };
1049
- }
1050
-
1051
- // Atomic swap path for workspace/*-carrying bundles.
1052
-
1053
- // Close the live SQLite connection so the DB file inside the real
1054
- // workspace can be replaced. The singleton lazily reopens on next use.
1055
- try {
1056
- resetDb();
1057
- } catch (err) {
1058
- // resetDb close failure is extremely unlikely but not worth aborting
1059
- // over — log and continue.
1060
- log.warn({ err }, "resetDb threw before swap; continuing");
1061
- }
1062
-
1063
- // Preserve the target's `cadet:*` credential metadata entries across
1064
- // the swap. Django's post-hatch provisioning writes
1065
- // `cadet:assistant_api_key` / `cadet:webhook_secret` via POST /v1/secrets,
1066
- // which upserts into the live
1067
- // workspace's `data/credentials/metadata.json`. Without this merge the
1068
- // swap would replace that file with the source's copy (which has no
1069
- // cadet entries on local sources), and the gateway's
1070
- // `readServiceCredentials` would stop finding the cloud API key.
1071
- //
1072
- // Executes in the temp workspace only — no effect on the live workspace
1073
- // — so a failure here leaves pre-swap state untouched. Any filesystem
1074
- // error is logged and degraded to a warning rather than aborting the
1075
- // import (credential loss is recoverable via reprovision; an aborted
1076
- // swap is a larger regression).
1077
- const liveMetadataPath = join(
1078
- realWorkspaceDir,
1079
- "data",
1080
- "credentials",
1081
- "metadata.json",
1082
- );
1083
- const tempMetadataPath = join(
1084
- tempWorkspaceDir,
1085
- "data",
1086
- "credentials",
1087
- "metadata.json",
1088
- );
1089
- try {
1090
- await mergeCredentialMetadataIntoTemp(
1091
- liveMetadataPath,
1092
- tempMetadataPath,
1093
- warnings,
1094
- );
1095
- } catch (err) {
1096
- log.warn(
1097
- { err, liveMetadataPath, tempMetadataPath },
1098
- "Credential metadata merge failed before swap",
1099
- );
1100
- warnings.push(
1101
- `Credential metadata merge failed: ${errMessage(err)}; cadet:* entries may not survive the import`,
1102
- );
1103
- }
1104
-
1105
- // Carry-over: for every path in WORKSPACE_PRESERVE_PATHS, if the bundle
1106
- // did NOT populate it inside the temp workspace but the LIVE workspace
1107
- // has it, move the live copy into the temp workspace at the same
1108
- // relative location. Without this step the atomic swap erases live
1109
- // user data (SQLite DB, Qdrant store, embedding-models cache,
1110
- // deprecated/ quarantine) whenever the bundle omits those paths —
1111
- // e.g. partial bundles carrying only prompts/config.
1112
- //
1113
- // Carry-over uses `rename` (not `cp`) to stay zero-disk on the happy
1114
- // path, which is critical on instances with multi-GB Qdrant stores or
1115
- // SQLite DBs and limited free space.
1116
- //
1117
- // Crash-safety is achieved in two phases:
1118
- // 1. `planCarryOverPreservedPaths` walks the live + temp trees WITHOUT
1119
- // mutating anything and produces the full intended `carried` list.
1120
- // 2. `writeImportMarker` persists that plan to disk BEFORE any rename
1121
- // runs. If the process dies during the subsequent
1122
- // `executeCarryOverPlan`, the marker already holds every
1123
- // (liveChild, tempChild) pair the next `recoverInterruptedImport`
1124
- // needs to replay. The marker is deleted only after the atomic
1125
- // swap pair succeeds (or in-process failure paths explicitly
1126
- // restore state).
1127
- let carried: CarriedPath[];
1128
- try {
1129
- carried = await planCarryOverPreservedPaths(
1130
- realWorkspaceDir,
1131
- tempWorkspaceDir,
1132
- );
1133
- } catch (err) {
1134
- await cleanupTempDir();
1135
- return {
1136
- ok: false,
1137
- reason: "write_failed",
1138
- message: `Failed to plan preserved-path carry-over: ${errMessage(err)}`,
1139
- };
1140
- }
1141
-
1142
- // Ensure the workspace dir exists so writeImportMarker (which writes
1143
- // at `<realWorkspaceDir>/.import-marker.json`) can land the file on
1144
- // first-ever imports where the workspace has never been created.
1145
- // mkdir is idempotent via { recursive: true }.
1146
- await mkdir(realWorkspaceDir, { recursive: true });
1147
-
1148
- const markerPath = importMarkerPathFor(realWorkspaceDir);
1149
- try {
1150
- await writeImportMarker(markerPath, {
1151
- tempWorkspaceDir,
1152
- carried: carried.map((c) => ({
1153
- liveChild: c.liveChild,
1154
- tempChild: c.tempChild,
1155
- })),
1156
- });
1157
- } catch (err) {
1158
- // Persisting the recovery plan is a prerequisite for crash-safe
1159
- // carry-over. If we can't write the marker, refuse to mutate the live
1160
- // workspace — a mid-carryover crash would otherwise be unrecoverable.
1161
- await cleanupTempDir();
1162
- return {
1163
- ok: false,
1164
- reason: "write_failed",
1165
- message: `Failed to persist import recovery marker: ${errMessage(err)}`,
1166
- };
1167
- }
1168
-
1169
- try {
1170
- await executeCarryOverPlan(carried);
1171
- } catch (err) {
1172
- // A rename in the plan failed. Restore the already-moved entries so
1173
- // the live workspace is whole again, then delete the marker and temp
1174
- // dir. `restoreCarriedPaths` is a no-op on entries that were never
1175
- // moved (tempChild missing), so passing the full plan is safe.
1176
- await restoreCarriedPaths(carried);
1177
- await safelyDeleteMarker(markerPath);
1178
- await cleanupTempDir();
1179
- return {
1180
- ok: false,
1181
- reason: "write_failed",
1182
- message: `Failed to carry over preserved workspace paths: ${errMessage(err)}`,
1183
- };
1184
- }
1185
-
1186
- // Workspace swap: content-level, not directory-level.
1187
- //
1188
- // We do NOT `rename(realWorkspaceDir, backupDir)` because in the
1189
- // production platform deployment `realWorkspaceDir` is a mounted volume
1190
- // (and the daemon's cwd / open subsystems pin it), so the kernel returns
1191
- // EBUSY on the parent-directory rename. Instead we swap the DIRECTORY'S
1192
- // CONTENTS: move every top-level entry from `realWorkspaceDir` into a
1193
- // peer `${realWorkspaceDir}.pre-import-<ts>/` backup dir, then move every
1194
- // top-level entry from the temp tree into the (now empty)
1195
- // `realWorkspaceDir`. `realWorkspaceDir` itself is never renamed, so
1196
- // mount-point / cwd pinning doesn't matter.
1197
- //
1198
- // Update the marker to record the backup dir BEFORE any move runs, so
1199
- // `recoverInterruptedImport` on a future boot can restore from backup
1200
- // even if the process is killed mid-swap.
1201
- // backupDir also lives INSIDE the workspace mount — same rationale as
1202
- // tempWorkspaceDir (keep all moves on the same filesystem, dot-prefix
1203
- // so workspace walkers skip it). Suffix with a UUID (not just a
1204
- // timestamp) so a malicious bundle can't guess the name and ship a
1205
- // top-level entry that collides with our active backup dir during
1206
- // phase 2 — phase 2 also rejects any such collision defensively.
1207
- const backupDir = join(
1208
- realWorkspaceDir,
1209
- `${IMPORT_BACKUP_PREFIX}${Date.now()}-${randomUUID()}`,
1210
- );
1211
- try {
1212
- await writeImportMarker(markerPath, {
1213
- tempWorkspaceDir,
1214
- carried: carried.map((c) => ({
1215
- liveChild: c.liveChild,
1216
- tempChild: c.tempChild,
1217
- })),
1218
- backupDir,
1219
- });
1220
- } catch (err) {
1221
- await restoreCarriedPaths(carried);
1222
- await cleanupTempDir();
1223
- await safelyDeleteMarker(markerPath);
1224
- return {
1225
- ok: false,
1226
- reason: "write_failed",
1227
- message: `Failed to persist pre-swap recovery marker: ${errMessage(err)}`,
1228
- };
1229
- }
1230
-
1231
- try {
1232
- await swapWorkspaceContents(realWorkspaceDir, tempWorkspaceDir, backupDir);
1233
-
1234
- // Swap succeeded. Record that fact in the marker BEFORE deleting it —
1235
- // otherwise a crash between `swapWorkspaceContents` returning and
1236
- // `safelyDeleteMarker` completing would leave a marker with
1237
- // `backupDir` populated, and `recoverInterruptedImport` on the next
1238
- // boot would silently roll back the successful import by restoring
1239
- // from backup. With `swapCompleted: true` the recovery path knows to
1240
- // skip the backup restore and just clean up residual artifacts.
1241
- try {
1242
- await writeImportMarker(markerPath, {
1243
- tempWorkspaceDir,
1244
- carried: carried.map((c) => ({
1245
- liveChild: c.liveChild,
1246
- tempChild: c.tempChild,
1247
- })),
1248
- backupDir,
1249
- swapCompleted: true,
1250
- });
1251
- } catch (err) {
1252
- // Very unlikely (we wrote it a moment ago) and not worth failing
1253
- // the whole import. A crash here would roll back via recovery, but
1254
- // the import itself is already applied.
1255
- log.warn(
1256
- { err, markerPath },
1257
- "Failed to mark import recovery marker as swapCompleted; crash window remains until safelyDeleteMarker",
1258
- );
1259
- }
1260
- await safelyDeleteMarker(markerPath);
1261
- } catch (err) {
1262
- // Content-level swap either rolled back its own renames (best effort)
1263
- // or left the workspace in an ambiguous state. Do a final restore pass
1264
- // from backupDir into realWorkspaceDir so any entries that didn't
1265
- // make it back end up whole again — the backup restore runs FIRST so
1266
- // it doesn't later clobber preserved paths that restoreCarriedPaths
1267
- // just put back. Pass the carried plan so restoreFromBackupDir can
1268
- // avoid clobbering descendants (e.g. `data/db` already restored
1269
- // under `data/`) when it replaces a top-level backup entry.
1270
- const restoreResult = await restoreFromBackupDir(
1271
- backupDir,
1272
- realWorkspaceDir,
1273
- carried,
1274
- );
1275
- await restoreCarriedPaths(carried);
1276
- if (restoreResult.ok) {
1277
- await cleanupTempDir();
1278
- await rm(backupDir, { recursive: true, force: true }).catch(() => {
1279
- /* best effort */
1280
- });
1281
- await safelyDeleteMarker(markerPath);
1282
- } else {
1283
- // Partial restore — preserve the backup dir, the temp tree, and the
1284
- // marker so an operator (or the next boot-time
1285
- // recoverInterruptedImport) can retry. The marker's `carried` plan
1286
- // references tempChild paths; deleting the temp tree here would
1287
- // break that plan. A backup dir with unresolved content is the last
1288
- // recoverable copy of the pre-import state.
1289
- log.error(
1290
- {
1291
- backupDir,
1292
- tempWorkspaceDir,
1293
- markerPath,
1294
- failedCount: restoreResult.failedCount,
1295
- },
1296
- "Pre-import backup restore incomplete; leaving backup dir, temp tree, and marker on disk for manual/boot-time recovery",
1297
- );
1298
- }
1299
- return {
1300
- ok: false,
1301
- reason: "write_failed",
1302
- message: `Failed to swap workspace contents: ${errMessage(err)}`,
1303
- };
1304
- }
1305
-
1306
- // -------------------------------------------------------------------------
1307
- // Post-commit side effects (non-fatal)
1308
- //
1309
- // Past this point the real workspace is already replaced — failures here
1310
- // do not justify reverting the whole import. Log loudly, surface warnings
1311
- // in the report, return success.
1312
- // -------------------------------------------------------------------------
1313
-
1314
- if (importCredentials && bufferedCredentials.length > 0) {
1315
- try {
1316
- await importCredentials(bufferedCredentials);
1317
- } catch (err) {
1318
- log.warn(
1319
- { err, count: bufferedCredentials.length },
1320
- "Post-commit credential import failed",
1321
- );
1322
- warnings.push(`Credential import failed: ${errMessage(err)}`);
1323
- }
1324
- }
1325
-
1326
- try {
1327
- invalidateConfigCache();
1328
- } catch (err) {
1329
- log.warn({ err }, "invalidateConfigCache threw after import");
1330
- }
1331
-
1332
- // Remove the backup dir (best-effort). Leaving it around is not a
1333
- // correctness issue, only a disk-space one, so we swallow errors. The
1334
- // backup dir now always exists once swap succeeds — we created it during
1335
- // swapWorkspaceContents to hold the pre-import live entries. Awaited so
1336
- // callers (and tests) observe a workspace free of `.pre-import-*`
1337
- // residue once this function returns.
1338
- await rm(backupDir, { recursive: true, force: true }).catch((err) => {
1339
- log.warn({ err, backupDir }, "Failed to remove pre-import backup dir");
1340
- });
1341
-
1342
- const report = buildReport(manifest, importedFiles, warnings);
1343
- return { ok: true, report };
1344
- }
1345
-
1346
- // ---------------------------------------------------------------------------
1347
- // Helpers
1348
- // ---------------------------------------------------------------------------
1349
-
1350
- function sha256Hex(data: Uint8Array): string {
1351
- return createHash("sha256").update(data).digest("hex");
1352
- }
1353
-
1354
- function generateBackupPath(diskPath: string): string {
1355
- const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
1356
- return `${diskPath}.backup-${timestamp}`;
1357
- }
1358
-
1359
- /**
1360
- * Promote verified-into-temp files for a legacy-format bundle into the
1361
- * live workspace in place. Mirrors commitImport's legacy write path:
1362
- *
1363
- * - If the live path already exists, copy it to a timestamped
1364
- * `${livePath}.backup-<ts>` sibling first.
1365
- * - Ensure the parent directory exists.
1366
- * - `fs.rename` the temp file over the live path for per-file atomicity.
1367
- * If that fails with EXDEV (cross-filesystem), fall back to `copyFile`
1368
- * then `rm` of the temp source.
1369
- * - Update the corresponding `ImportedFileReport` with the overwrite
1370
- * action and backup path so the report matches commitImport's output.
1371
- */
1372
- async function promoteLegacyStagedFiles(
1373
- staged: Array<{
1374
- tempPath: string;
1375
- livePath: string;
1376
- archivePath: string;
1377
- importedFileIndex: number;
1378
- }>,
1379
- importedFiles: ImportedFileReport[],
1380
- ): Promise<void> {
1381
- for (const entry of staged) {
1382
- // Backup before overwrite, matching commitImport.
1383
- //
1384
- // Use lstat (not existsSync) to detect a pre-existing entry: existsSync
1385
- // follows symlinks, so a dangling pre-existing symlink at livePath would
1386
- // report `false` and we'd skip the backup before later atomically
1387
- // replacing it via rename.
1388
- let preExisting: boolean;
1389
- try {
1390
- await lstat(entry.livePath);
1391
- preExisting = true;
1392
- } catch (err) {
1393
- if (isENOENT(err)) {
1394
- preExisting = false;
1395
- } else {
1396
- throw err;
1397
- }
1398
- }
1399
-
1400
- let backupPath: string | null = null;
1401
- if (preExisting) {
1402
- backupPath = generateBackupPath(entry.livePath);
1403
- // copyFile follows symlinks and copies the resolved file's content, so
1404
- // backing up a pre-existing symlink with copyFile would lose the
1405
- // symlink shape. Recreate the link via readlink + symlink instead;
1406
- // fall back to copyFile for regular files.
1407
- const liveStat = await lstat(entry.livePath);
1408
- if (liveStat.isSymbolicLink()) {
1409
- const target = await readlink(entry.livePath);
1410
- await symlink(target, backupPath);
1411
- } else {
1412
- await copyFile(entry.livePath, backupPath);
1413
- }
1414
- }
1415
-
1416
- await mkdir(dirname(entry.livePath), { recursive: true });
1417
-
1418
- // If we're replacing a SQLite main database file, remove any sibling
1419
- // `.db-wal`/`.db-shm`/`.db-journal` from live first. Those
1420
- // auxiliary files are only valid with the exact `.db` that wrote
1421
- // them — leaving them alongside the replacement DB causes SQLite to
1422
- // replay incompatible WAL frames on the first open and report
1423
- // "database disk image is malformed".
1424
- if (entry.livePath.endsWith(".db")) {
1425
- for (const suffix of [".db-wal", ".db-shm", ".db-journal"]) {
1426
- const auxPath = `${entry.livePath.slice(0, -".db".length)}${suffix}`;
1427
- await rm(auxPath, { force: true }).catch(() => {
1428
- /* best effort */
1429
- });
1430
- }
1431
- }
1432
-
1433
- try {
1434
- await rename(entry.tempPath, entry.livePath);
1435
- } catch (err) {
1436
- if (isEXDEV(err)) {
1437
- // copyFile follows symlinks and copies the target's CONTENT — so a
1438
- // legacy-format symlink entry (e.g. `prompts/USER.md` encoded as a
1439
- // typeflag-2 record) would land as a regular file containing the
1440
- // linked target's bytes. lstat the source first; if it's a symlink,
1441
- // recreate the symlink shape via readlink + symlink. Mirrors the
1442
- // verbatimSymlinks: true contract that copyTreeSkippingTransient
1443
- // already uses on the atomic-swap path.
1444
- const srcStat = await lstat(entry.tempPath);
1445
- if (srcStat.isSymbolicLink()) {
1446
- const target = await readlink(entry.tempPath);
1447
- // Unlike rename (which atomically overwrites), fs.promises.symlink
1448
- // fails with EEXIST if the destination already exists. Remove any
1449
- // pre-existing entry at livePath first — the backup above
1450
- // preserved its contents.
1451
- await rm(entry.livePath, { force: true });
1452
- await symlink(target, entry.livePath);
1453
- } else {
1454
- await copyFile(entry.tempPath, entry.livePath);
1455
- }
1456
- await rm(entry.tempPath, { force: true });
1457
- } else {
1458
- throw err;
1459
- }
1460
- }
1461
-
1462
- const report = importedFiles[entry.importedFileIndex];
1463
- if (report) {
1464
- if (backupPath) {
1465
- report.action = "overwritten";
1466
- report.backup_path = backupPath;
1467
- } else {
1468
- report.action = "created";
1469
- }
1470
- }
1471
- }
1472
- }
1473
-
1474
- /**
1475
- * Rewrite the temp workspace's `data/credentials/metadata.json` so the
1476
- * target's live `cadet:*` entries survive the swap. Exits silently if
1477
- * there is nothing to merge.
1478
- *
1479
- * Four cases:
1480
- * - No live metadata, no temp metadata → no-op.
1481
- * - Live metadata present, temp metadata missing → if the live metadata
1482
- * contains cadet entries, synthesize a minimal v5 metadata file in
1483
- * the temp tree containing only those preserved entries. If it has
1484
- * none, no-op (no entries to preserve).
1485
- * - Live metadata missing, temp metadata present → no-op (nothing to
1486
- * preserve; the bundle's copy lands as-is).
1487
- * - Both present → run the merge helper and rewrite the temp copy.
1488
- *
1489
- * Invoked under a try/catch by the caller; thrown errors surface as
1490
- * warnings but don't abort the import.
1491
- */
1492
- async function mergeCredentialMetadataIntoTemp(
1493
- liveMetadataPath: string,
1494
- tempMetadataPath: string,
1495
- warnings: string[],
1496
- ): Promise<void> {
1497
- let liveJson: string | null = null;
1498
- try {
1499
- liveJson = await readFile(liveMetadataPath, "utf-8");
1500
- } catch (err) {
1501
- if (!isENOENT(err)) {
1502
- throw err;
1503
- }
1504
- }
1505
-
1506
- let tempJson: string | null = null;
1507
- try {
1508
- tempJson = await readFile(tempMetadataPath, "utf-8");
1509
- } catch (err) {
1510
- if (!isENOENT(err)) {
1511
- throw err;
1512
- }
1513
- }
1514
-
1515
- if (liveJson == null && tempJson == null) {
1516
- return;
1517
- }
1518
-
1519
- if (tempJson != null) {
1520
- const merged = mergeMetadataPreservingCadet(tempJson, liveJson);
1521
- if (merged !== tempJson) {
1522
- await writeFile(tempMetadataPath, merged, { mode: 0o600 });
1523
- }
1524
- return;
1525
- }
1526
-
1527
- // Live-only path: synthesize a v5 file with just the preserved cadet
1528
- // entries so the gateway can still locate them after the swap.
1529
- const synthesized = mergeMetadataPreservingCadet(
1530
- JSON.stringify({ version: 5, credentials: [] }),
1531
- liveJson,
1532
- );
1533
- const parsed = z
1534
- .object({ credentials: z.array(z.unknown()).optional() })
1535
- .safeParse(JSON.parse(synthesized));
1536
- if (!parsed.success || (parsed.data.credentials?.length ?? 0) === 0) {
1537
- // Live file exists but had no cadet entries worth preserving.
1538
- return;
1539
- }
1540
-
1541
- try {
1542
- await mkdir(dirname(tempMetadataPath), { recursive: true });
1543
- await writeFile(tempMetadataPath, synthesized, { mode: 0o600 });
1544
- } catch (err) {
1545
- warnings.push(
1546
- `Failed to write preserved cadet:* metadata into temp workspace: ${errMessage(err)}`,
1547
- );
1548
- }
1549
- }
1550
-
1551
- function buildReport(
1552
- manifest: ManifestType,
1553
- files: ImportedFileReport[],
1554
- warnings: string[],
1555
- ): ImportCommitReport {
1556
- return {
1557
- success: true,
1558
- summary: {
1559
- total_files: files.length,
1560
- files_created: files.filter((f) => f.action === "created").length,
1561
- files_overwritten: files.filter((f) => f.action === "overwritten").length,
1562
- files_skipped: files.filter((f) => f.action === "skipped").length,
1563
- backups_created: files.filter((f) => f.backup_path !== null).length,
1564
- },
1565
- files,
1566
- manifest,
1567
- warnings,
1568
- };
1569
- }
1570
-
1571
- /**
1572
- * Copy any WORKSPACE_PRESERVE_PATHS entries from the live workspace into
1573
- * the temp workspace when the bundle did not already populate them. Runs
1574
- * immediately before the atomic swap so the swap-in tree has the union
1575
- * of bundle-provided files and live-preserved files.
1576
- *
1577
- * Per-file merge semantics (critical): a bundle that touches a SINGLE file
1578
- * under a preserved directory (e.g. writes `workspace/data/qdrant/config.json`)
1579
- * must NOT cause the rest of that directory to be wiped. We therefore walk
1580
- * each preserved path recursively and carry over any live file or
1581
- * subdirectory the bundle did not itself write. A whole-directory short-
1582
- * circuit would mis-handle that case by erasing unrelated qdrant segments,
1583
- * DB WALs, embedding-model shards, etc.
1584
- *
1585
- * For each preserved relative path:
1586
- * - If the preserved path is a FILE in the live workspace and the temp
1587
- * tree already has that exact path, the bundle populated it — leave
1588
- * it alone. Otherwise rename/copy the live file over.
1589
- * - If the preserved path is a DIRECTORY in the live workspace, walk
1590
- * it recursively. For each entry:
1591
- * * If the temp tree has a matching entry at the same relative
1592
- * path, the bundle wrote it — skip.
1593
- * * If not, carry the live entry over (rename with EXDEV fallback
1594
- * to recursive copy).
1595
- * The walk stops descending on any subtree the bundle has completely
1596
- * populated, since we only need to fill gaps.
1597
- */
1598
- /**
1599
- * Pre-compute the full `CarriedPath[]` that `carryOverPreservedPaths` will
1600
- * move, WITHOUT mutating the live workspace. The result lets us write the
1601
- * crash-recovery marker before any rename runs, so a crash mid-carry-over
1602
- * still leaves a complete restoration plan for the next
1603
- * `recoverInterruptedImport` call.
1604
- *
1605
- * The walk mirrors `carryOverPreservedPaths` exactly — if the two were to
1606
- * disagree, recovery would be incomplete. Directory subtrees that the
1607
- * bundle didn't populate are recorded as a single top-level move (matches
1608
- * the one-shot rename the executor does); per-file merges happen otherwise.
1609
- */
1610
- async function planCarryOverPreservedPaths(
1611
- realWorkspaceDir: string,
1612
- tempWorkspaceDir: string,
1613
- ): Promise<CarriedPath[]> {
1614
- const plan: CarriedPath[] = [];
1615
- for (const rel of policy.WORKSPACE_PRESERVE_PATHS) {
1616
- const livePath = join(realWorkspaceDir, rel);
1617
- const tempPath = join(tempWorkspaceDir, rel);
1618
-
1619
- let liveStat;
1620
- try {
1621
- liveStat = await stat(livePath);
1622
- } catch (err) {
1623
- if (isENOENT(err)) {
1624
- continue;
1625
- }
1626
- throw err;
1627
- }
1628
-
1629
- if (!liveStat.isDirectory()) {
1630
- if (existsSync(tempPath)) {
1631
- continue;
1632
- }
1633
- plan.push({ liveChild: livePath, tempChild: tempPath });
1634
- continue;
1635
- }
1636
-
1637
- await planMergeLiveIntoTempDir(livePath, tempPath, plan);
1638
- }
1639
- return plan;
1640
- }
1641
-
1642
- /**
1643
- * Same walk as `mergeLiveIntoTempDir` but only records the would-be moves
1644
- * in `plan`. Intentionally side-effect-free apart from appending to the
1645
- * plan array.
1646
- */
1647
- async function planMergeLiveIntoTempDir(
1648
- liveDir: string,
1649
- tempDir: string,
1650
- plan: CarriedPath[],
1651
- ): Promise<void> {
1652
- let entries;
1653
- try {
1654
- entries = await readdir(liveDir, { withFileTypes: true });
1655
- } catch (err) {
1656
- if (isENOENT(err)) {
1657
- return;
1658
- }
1659
- throw err;
1660
- }
1661
-
1662
- for (const entry of entries) {
1663
- const liveChild = join(liveDir, entry.name);
1664
- const tempChild = join(tempDir, entry.name);
1665
- const existsInTemp = existsSync(tempChild);
1666
-
1667
- if (entry.isDirectory()) {
1668
- if (!existsInTemp) {
1669
- plan.push({ liveChild, tempChild });
1670
- continue;
1671
- }
1672
- await planMergeLiveIntoTempDir(liveChild, tempChild, plan);
1673
- continue;
1674
- }
1675
-
1676
- if (existsInTemp) {
1677
- continue;
1678
- }
1679
-
1680
- // SQLite auxiliary files (WAL / SHM / journal) are only valid as a
1681
- // pair with the exact `.db` they were written by. If the bundle
1682
- // replaced the sibling `.db` in this dir, carrying the live `.db-wal`
1683
- // forward pairs stale WAL frames with a different DB and SQLite
1684
- // reports "database disk image is malformed" on first open. Drop
1685
- // them — SQLite recreates a fresh WAL lazily on next connection,
1686
- // and the export already checkpointed the source WAL into the main
1687
- // DB before the bundle was built.
1688
- //
1689
- // When the bundle does NOT carry a replacement DB (bundle is
1690
- // config-only etc.), the live `.db` is preserved and the live WAL
1691
- // stays paired with it.
1692
- if (
1693
- isSqliteAuxiliaryFile(entry.name) &&
1694
- hasSiblingDbInTemp(tempDir, entry.name)
1695
- ) {
1696
- continue;
1697
- }
1698
-
1699
- plan.push({ liveChild, tempChild });
1700
- }
1701
- }
1702
-
1703
- /**
1704
- * SQLite writes `<name>.db-wal`, `<name>.db-shm`, `<name>.db-journal`
1705
- * alongside its main `<name>.db` file. These are only consistent with
1706
- * the exact `.db` they were created for.
1707
- */
1708
- function isSqliteAuxiliaryFile(name: string): boolean {
1709
- return (
1710
- name.endsWith(".db-wal") ||
1711
- name.endsWith(".db-shm") ||
1712
- name.endsWith(".db-journal")
1713
- );
1714
- }
1715
-
1716
- /**
1717
- * Does the temp dir contain the main `.db` file that owns this auxiliary
1718
- * file? Given e.g. `assistant.db-wal`, checks for `tempDir/assistant.db`.
1719
- */
1720
- function hasSiblingDbInTemp(tempDir: string, auxName: string): boolean {
1721
- const dbName = auxName
1722
- .replace(/\.db-wal$/, ".db")
1723
- .replace(/\.db-shm$/, ".db")
1724
- .replace(/\.db-journal$/, ".db");
1725
- return existsSync(join(tempDir, dbName));
1726
- }
1727
-
1728
- /**
1729
- * Execute a carry-over plan produced by `planCarryOverPreservedPaths`.
1730
- * Each entry is moved with `carryOverEntry`; directories that are plan
1731
- * roots have their parent created so `rename` can land them.
1732
- *
1733
- * Per-entry failures abort the loop and throw — the caller is expected to
1734
- * run `restoreCarriedPaths` on the already-moved entries (a subset of the
1735
- * plan) on its in-process failure path.
1736
- */
1737
- async function executeCarryOverPlan(plan: CarriedPath[]): Promise<void> {
1738
- for (const { liveChild, tempChild } of plan) {
1739
- await mkdir(dirname(tempChild), { recursive: true });
1740
- await carryOverEntry(liveChild, tempChild);
1741
- }
1742
- }
1743
-
1744
- /**
1745
- * Move a single live workspace entry (file or directory) into the temp
1746
- * workspace. Uses `rename` for the fast path (same-filesystem, zero copy)
1747
- * so we don't duplicate potentially multi-GB preserved trees like
1748
- * `data/qdrant` or `data/db`. Falls back to `cp` + `rm` on EXDEV (different
1749
- * filesystems) — rare in practice since live and temp share a parent dir.
1750
- *
1751
- * Live data is moved, not copied, so the atomic swap must restore it on
1752
- * failure. `streamCommitImport` tracks every carry-over via `CarriedPath`
1753
- * and calls `restoreCarriedPaths` on any swap-pair error so the live
1754
- * workspace ends up whole even if the import aborts.
1755
- */
1756
- async function carryOverEntry(
1757
- liveChild: string,
1758
- tempChild: string,
1759
- ): Promise<void> {
1760
- try {
1761
- await rename(liveChild, tempChild);
1762
- } catch (err) {
1763
- if (isEXDEV(err)) {
1764
- await copyTreeSkippingTransient(liveChild, tempChild);
1765
- await rm(liveChild, { recursive: true, force: true });
1766
- } else {
1767
- throw err;
1768
- }
1769
- }
1770
- }
1771
-
1772
- /**
1773
- * Every preserved entry that was moved out of the live workspace during
1774
- * carry-over. Used to undo the move if the atomic swap fails, so we never
1775
- * leave the daemon with SQLite/Qdrant/embedding-model data stranded in a
1776
- * temp tree that's about to be deleted.
1777
- */
1778
- interface CarriedPath {
1779
- /** Original location inside the live workspace (real path before swap). */
1780
- liveChild: string;
1781
- /** Landing location inside the temp workspace. */
1782
- tempChild: string;
1783
- }
1784
-
1785
- /**
1786
- * Undo a set of carry-over moves by renaming each carried path back to its
1787
- * original live location. Best-effort: logs and continues on per-entry
1788
- * failures rather than throwing, since the caller is already handling a
1789
- * swap-pair failure and needs to restore as much state as possible.
1790
- */
1791
- async function restoreCarriedPaths(
1792
- carried: readonly CarriedPath[],
1793
- ): Promise<void> {
1794
- for (const { liveChild, tempChild } of carried) {
1795
- try {
1796
- await mkdir(dirname(liveChild), { recursive: true });
1797
- await rename(tempChild, liveChild);
1798
- } catch (err) {
1799
- if (isEXDEV(err)) {
1800
- try {
1801
- await copyTreeSkippingTransient(tempChild, liveChild);
1802
- await rm(tempChild, { recursive: true, force: true });
1803
- continue;
1804
- } catch (cpErr) {
1805
- log.error(
1806
- { err: cpErr, liveChild, tempChild },
1807
- "Failed to restore carried preserved path via cp fallback; manual recovery may be required",
1808
- );
1809
- continue;
1810
- }
1811
- }
1812
- if (isENOENT(err)) {
1813
- // The entry may have already moved (rename-pair partially succeeded)
1814
- // or never existed. Nothing to restore.
1815
- continue;
1816
- }
1817
- log.error(
1818
- { err, liveChild, tempChild },
1819
- "Failed to restore carried preserved path; manual recovery may be required",
1820
- );
1821
- }
1822
- }
1823
- }
1824
-
1825
- /**
1826
- * Swap the CONTENTS of the workspace without ever renaming `realWorkspaceDir`
1827
- * itself. The production platform pod has `realWorkspaceDir` as a mounted
1828
- * volume (and the daemon's subsystems pin file handles inside it), so the
1829
- * kernel returns `EBUSY` if we `rename()` the directory. Moving individual
1830
- * top-level entries sidesteps that: a mount point's children usually aren't
1831
- * themselves mount points, and individual-file EBUSY is much rarer than
1832
- * directory-rename EBUSY.
1833
- *
1834
- * Semantics:
1835
- *
1836
- * 1. Create `backupDir` (peer of `realWorkspaceDir`, different parent entry).
1837
- * 2. For each top-level entry currently in `realWorkspaceDir`, `rename()`
1838
- * it into `backupDir`.
1839
- * 3. For each top-level entry in `tempWorkspaceDir`, `rename()` it into
1840
- * `realWorkspaceDir`.
1841
- * 4. Remove the (now empty) temp dir.
1842
- *
1843
- * On a per-entry rename failure during phase 2, move what was already moved
1844
- * back into `realWorkspaceDir` and throw. On a failure during phase 3, move
1845
- * the already-moved temp entries back to `tempWorkspaceDir`, then move
1846
- * backup entries back into `realWorkspaceDir`, and throw.
1847
- *
1848
- * This function is NOT atomic — a reader that opens `realWorkspaceDir`
1849
- * mid-swap will see a half-emptied state. The daemon's SQLite connection is
1850
- * already closed (`resetDb()` ran before this), and the async import is
1851
- * running in a background job from the external caller's perspective, so
1852
- * transient readers aren't expected. `recoverInterruptedImport` uses the
1853
- * `backupDir` recorded in the marker to finish the rollback if a crash hits
1854
- * mid-swap.
1855
- */
1856
- async function swapWorkspaceContents(
1857
- realWorkspaceDir: string,
1858
- tempWorkspaceDir: string,
1859
- backupDir: string,
1860
- ): Promise<void> {
1861
- // Symlinks in the temp tree pass through unchanged: `rename` moves the
1862
- // symlink inode without dereferencing, and the EXDEV fallback (`fs.cp`
1863
- // with `verbatimSymlinks: true`) preserves them too.
1864
- await mkdir(backupDir, { recursive: true });
1865
-
1866
- // Phase 1: move every top-level entry out of real into backup. Skip
1867
- // ONLY the exact scratch dirs this import owns (backupDir itself, and
1868
- // the tempWorkspaceDir passed in) — NOT everything that happens to
1869
- // start with the `.import-`/`.pre-import-` prefix. A user workspace
1870
- // that legitimately contains an entry with one of those prefixes
1871
- // would otherwise leak state across imports, and a bundle carrying
1872
- // the same name would collide on phase-2 rename-in.
1873
- //
1874
- // The recovery marker (`.import-marker.json`) is also reserved — it
1875
- // lives inside the workspace, must stay put across the swap so
1876
- // recovery can read it if the process dies mid-swap, and must not be
1877
- // overwritten by a bundle entry of the same name.
1878
- const scratchBasenames = new Set<string>([
1879
- basename(backupDir),
1880
- basename(tempWorkspaceDir),
1881
- IMPORT_MARKER_BASENAME,
1882
- ]);
1883
- let liveEntries: string[];
1884
- try {
1885
- liveEntries = (await readdir(realWorkspaceDir)).filter(
1886
- (name) => !scratchBasenames.has(name),
1887
- );
1888
- } catch (err) {
1889
- if (isENOENT(err)) {
1890
- liveEntries = [];
1891
- } else {
1892
- throw err;
1893
- }
1894
- }
1895
-
1896
- const movedToBackup: string[] = [];
1897
- try {
1898
- for (const name of liveEntries) {
1899
- await moveEntryWithExdevFallback(
1900
- join(realWorkspaceDir, name),
1901
- join(backupDir, name),
1902
- );
1903
- movedToBackup.push(name);
1904
- }
1905
- } catch (err) {
1906
- // Partial move-out. Reverse what we moved so realWorkspaceDir ends up
1907
- // back to its original content before we throw.
1908
- for (const name of movedToBackup.reverse()) {
1909
- try {
1910
- await moveEntryWithExdevFallback(
1911
- join(backupDir, name),
1912
- join(realWorkspaceDir, name),
1913
- );
1914
- } catch (restoreErr) {
1915
- log.error(
1916
- { err: restoreErr, name, realWorkspaceDir, backupDir },
1917
- "Failed to restore entry from backup during swap-out rollback",
1918
- );
1919
- }
1920
- }
1921
- throw err;
1922
- }
1923
-
1924
- // Phase 2: move every top-level entry from temp into real. `rename`
1925
- // requires the destination's parent to exist, so ensure realWorkspaceDir
1926
- // exists even if phase 1 found no entries (first-ever import into a
1927
- // fresh workspace dir that hasn't been created yet).
1928
- await mkdir(realWorkspaceDir, { recursive: true });
1929
-
1930
- let tempEntries: string[];
1931
- try {
1932
- tempEntries = await readdir(tempWorkspaceDir);
1933
- } catch (err) {
1934
- // A missing temp tree here is a hard failure — phase 1 has already
1935
- // emptied realWorkspaceDir into backup, so treating temp as an empty
1936
- // import would commit an empty workspace and the backup would be
1937
- // deleted in the success path. That's silent data loss. Throw so the
1938
- // caller's rollback restores backup → real.
1939
- if (isENOENT(err)) {
1940
- throw new Error(
1941
- `Temp workspace dir disappeared before swap-in (${tempWorkspaceDir})`,
1942
- );
1943
- }
1944
- throw err;
1945
- }
1946
-
1947
- // Defend against a bundle whose top-level entries collide with this
1948
- // swap's scratch basenames. The UUID suffix on `backupDir` makes an
1949
- // accidental collision astronomically unlikely, but a malicious or
1950
- // corrupted bundle carrying e.g. `.pre-import-<exact-match>` could
1951
- // otherwise replace the (empty) active backup dir via rename on an
1952
- // empty live workspace, and the success-path `rm(backupDir)` would
1953
- // then silently delete the imported content. Fail fast before any
1954
- // rename so real ends up rolled back to pre-import state.
1955
- const collidingName = tempEntries.find((name) => scratchBasenames.has(name));
1956
- if (collidingName !== undefined) {
1957
- throw new Error(
1958
- `Bundle top-level entry "${collidingName}" collides with an import scratch dir basename — refusing to swap to avoid accidental deletion of imported content`,
1959
- );
1960
- }
1961
-
1962
- const movedToReal: string[] = [];
1963
- try {
1964
- for (const name of tempEntries) {
1965
- await moveEntryWithExdevFallback(
1966
- join(tempWorkspaceDir, name),
1967
- join(realWorkspaceDir, name),
1968
- );
1969
- movedToReal.push(name);
1970
- }
1971
- } catch (err) {
1972
- // Partial move-in. Reverse the partial fill-in first (real → temp),
1973
- // then restore from backup (backup → real), so real ends up back at
1974
- // its pre-swap state.
1975
- for (const name of movedToReal.reverse()) {
1976
- try {
1977
- await moveEntryWithExdevFallback(
1978
- join(realWorkspaceDir, name),
1979
- join(tempWorkspaceDir, name),
1980
- );
1981
- } catch (restoreErr) {
1982
- log.error(
1983
- { err: restoreErr, name, realWorkspaceDir, tempWorkspaceDir },
1984
- "Failed to undo partial swap-in during rollback",
1985
- );
1986
- }
1987
- }
1988
- for (const name of movedToBackup.reverse()) {
1989
- try {
1990
- await moveEntryWithExdevFallback(
1991
- join(backupDir, name),
1992
- join(realWorkspaceDir, name),
1993
- );
1994
- } catch (restoreErr) {
1995
- log.error(
1996
- { err: restoreErr, name, realWorkspaceDir, backupDir },
1997
- "Failed to restore entry from backup during swap-in rollback",
1998
- );
1999
- }
2000
- }
2001
- throw err;
2002
- }
2003
-
2004
- // Phase 3: remove the now-empty temp dir. If it still has stragglers
2005
- // (pax headers, etc. we didn't move) take them down too.
2006
- await rm(tempWorkspaceDir, { recursive: true, force: true }).catch(() => {
2007
- /* best effort — caller will log if it matters */
2008
- });
2009
- }
2010
-
2011
- /**
2012
- * Move a single filesystem entry from `src` to `dst`, falling back to
2013
- * `cp` + `rm` when `rename` returns EXDEV (cross-filesystem move).
2014
- *
2015
- * In the production container, `realWorkspaceDir` is typically a mounted
2016
- * volume on a separate filesystem from the backup / temp dirs that live on
2017
- * the overlay root — so every move in `swapWorkspaceContents` crosses a
2018
- * filesystem boundary and would fail with EXDEV without this fallback.
2019
- * Every other move helper in this file (`carryOverEntry`,
2020
- * `restoreCarriedPaths`, `restoreFromBackupDir`, `mergeBackupIntoLive`)
2021
- * already handles EXDEV the same way; this helper centralises that
2022
- * behaviour for the swap path.
2023
- */
2024
- async function moveEntryWithExdevFallback(
2025
- src: string,
2026
- dst: string,
2027
- ): Promise<void> {
2028
- try {
2029
- await rename(src, dst);
2030
- } catch (err) {
2031
- if (isEXDEV(err)) {
2032
- try {
2033
- await copyTreeSkippingTransient(src, dst);
2034
- } catch (cpErr) {
2035
- // Partial cp could leave incomplete content at `dst`. Remove it so
2036
- // `restoreFromBackupDir` (running on a later error path) doesn't
2037
- // mistake half-a-tree for a valid backup entry and clobber the
2038
- // still-intact source with it. Leave `src` alone — we never got
2039
- // to the rm step, so it's whole.
2040
- await rm(dst, { recursive: true, force: true }).catch((rmErr) => {
2041
- log.warn(
2042
- { err: rmErr, dst },
2043
- "Failed to clean up partial cp destination after EXDEV fallback failure",
2044
- );
2045
- });
2046
- throw cpErr;
2047
- }
2048
- await rm(src, { recursive: true, force: true });
2049
- return;
2050
- }
2051
- throw err;
2052
- }
2053
- }
2054
-
2055
- /**
2056
- * `fs.cp(..., { recursive: true })` throws `ERR_FS_CP_SOCKET` (and
2057
- * similar for FIFOs / other special files) in newer Node versions, which
2058
- * breaks imports in real deployments — most concretely, the meet-join
2059
- * skill creates unix sockets under `meets/<id>/sockets/` that end up
2060
- * inside the workspace. Special files are session-scoped, always safe to
2061
- * drop across an import. This wrapper asks `fs.cp` to skip anything that
2062
- * isn't a regular file / directory / symlink, and falls back to a manual
2063
- * walk if `fs.cp` still trips over something we couldn't filter ahead of
2064
- * time.
2065
- */
2066
- async function copyTreeSkippingTransient(
2067
- src: string,
2068
- dst: string,
2069
- ): Promise<void> {
2070
- try {
2071
- await cp(src, dst, {
2072
- recursive: true,
2073
- preserveTimestamps: true,
2074
- // Preserve symlinks instead of dereferencing. Without this, an
2075
- // EXDEV-fallback copy of a tree containing a class-1 symlink would
2076
- // resolve the symlink to its target's bytes — wrong both for the
2077
- // streaming importer's symlink entries (which must land on disk as
2078
- // real symlinks) and for any pre-existing symlinks in carried
2079
- // preserved subtrees.
2080
- verbatimSymlinks: true,
2081
- filter: async (source) => {
2082
- try {
2083
- const info = await lstat(source);
2084
- // Keep regular files, directories, and symlinks. Skip sockets,
2085
- // FIFOs, block/char devices — transient / non-portable content
2086
- // that `fs.cp` refuses to replicate anyway.
2087
- return info.isFile() || info.isDirectory() || info.isSymbolicLink();
2088
- } catch {
2089
- // If we can't stat, let `fs.cp` try and surface the real error.
2090
- return true;
2091
- }
2092
- },
2093
- });
2094
- } catch (err) {
2095
- if (!isCpUnsupportedFileType(err)) {
2096
- throw err;
2097
- }
2098
- // Fall back to a manual walk that skips anything that isn't a file,
2099
- // dir, or symlink. `fs.cp` on Node can still occasionally surface
2100
- // ERR_FS_CP_SOCKET despite the filter (races where the socket
2101
- // appears between filter call and read), so the manual walk is the
2102
- // last-resort path.
2103
- log.warn(
2104
- { err, src, dst },
2105
- "cp filter still surfaced unsupported file type; falling back to manual walk",
2106
- );
2107
- await manualCopyTreeSkippingTransient(src, dst);
2108
- }
2109
- }
2110
-
2111
- function isCpUnsupportedFileType(err: unknown): boolean {
2112
- if (!err || typeof err !== "object") {
2113
- return false;
2114
- }
2115
- const code = (err as { code?: string }).code;
2116
- return (
2117
- code === "ERR_FS_CP_SOCKET" ||
2118
- code === "ERR_FS_CP_FIFO_PIPE" ||
2119
- code === "ERR_FS_CP_UNKNOWN"
2120
- );
2121
- }
2122
-
2123
- async function manualCopyTreeSkippingTransient(
2124
- src: string,
2125
- dst: string,
2126
- ): Promise<void> {
2127
- const info = await lstat(src);
2128
- if (info.isSymbolicLink()) {
2129
- const target = await readlink(src);
2130
- await mkdir(dirname(dst), { recursive: true });
2131
- // `symlink` throws EEXIST if `dst` already exists. We may be running
2132
- // as a fallback after `fs.cp` partially populated `dst` (including
2133
- // creating this symlink itself), so clear it first — unlike
2134
- // `copyFile` / recursive `mkdir`, `symlink` has no replace-mode.
2135
- await rm(dst, { force: true }).catch(() => {
2136
- /* best effort — a subsequent symlink error will surface any real issue */
2137
- });
2138
- await symlink(target, dst);
2139
- return;
2140
- }
2141
- if (info.isFile()) {
2142
- await mkdir(dirname(dst), { recursive: true });
2143
- await copyFile(src, dst);
2144
- return;
2145
- }
2146
- if (info.isDirectory()) {
2147
- await mkdir(dst, { recursive: true });
2148
- for (const name of await readdir(src)) {
2149
- await manualCopyTreeSkippingTransient(join(src, name), join(dst, name));
2150
- }
2151
- return;
2152
- }
2153
- // Anything else (socket, FIFO, device) — intentionally skip.
2154
- log.debug(
2155
- { src },
2156
- "Skipping transient/special filesystem entry during cross-fs copy",
2157
- );
2158
- }
2159
-
2160
- interface RestoreFromBackupResult {
2161
- ok: boolean;
2162
- /** Entries that could not be restored; backup must be preserved if non-zero. */
2163
- failedCount: number;
2164
- }
2165
-
2166
- /**
2167
- * Move every top-level entry from `backupDir` back into `realWorkspaceDir`,
2168
- * overwriting partial swap-in leftovers from a crashed import.
2169
- *
2170
- * `carried` is the carry-over plan. Any entry in `carried` whose
2171
- * `liveChild` is a descendant of a backup entry protects that subtree from
2172
- * being rm'd — if the backup captured only part of a directory (because
2173
- * carry-over already moved `data/db` out before the swap started), we must
2174
- * not clobber a `data/db` that recovery already restored into
2175
- * `realWorkspaceDir/data/`. In that case we merge the backup's `data/`
2176
- * into `realWorkspaceDir/data/` per-entry instead of replacing it.
2177
- *
2178
- * Used by the in-process rollback path (failed `swapWorkspaceContents`)
2179
- * and by `recoverInterruptedImport` at boot.
2180
- *
2181
- * Best-effort per-entry: logs failures and continues rather than
2182
- * throwing, and returns a status with `failedCount` so callers can decide
2183
- * whether to preserve the backup dir for manual recovery. A missing
2184
- * backup dir is a clean no-op (`{ ok: true, failedCount: 0 }`).
2185
- */
2186
- async function restoreFromBackupDir(
2187
- backupDir: string,
2188
- realWorkspaceDir: string,
2189
- carried: readonly CarriedPath[],
2190
- ): Promise<RestoreFromBackupResult> {
2191
- let backupEntries: string[];
2192
- try {
2193
- backupEntries = await readdir(backupDir);
2194
- } catch (err) {
2195
- if (isENOENT(err)) {
2196
- return { ok: true, failedCount: 0 };
2197
- }
2198
- log.error(
2199
- { err, backupDir },
2200
- "Failed to read backup dir during restore; skipping backup restoration",
2201
- );
2202
- return { ok: false, failedCount: 1 };
2203
- }
2204
-
2205
- const carriedLivePaths = carried.map((c) => resolve(c.liveChild));
2206
-
2207
- let failedCount = 0;
2208
-
2209
- for (const name of backupEntries) {
2210
- const src = join(backupDir, name);
2211
- const dst = join(realWorkspaceDir, name);
2212
- const dstAbs = resolve(dst);
2213
-
2214
- // If any carried path lives strictly inside `dst` (e.g., dst is
2215
- // `real/data/` and a carried path is `real/data/db`), we can't
2216
- // wholesale `rm(dst) + rename(src)` — that would destroy the carried
2217
- // content that recovery has already put back. Merge instead.
2218
- const hasProtectedDescendant = carriedLivePaths.some((carriedAbs) => {
2219
- if (carriedAbs === dstAbs) {
2220
- return false;
2221
- }
2222
- return carriedAbs.startsWith(dstAbs + sep);
2223
- });
2224
-
2225
- if (hasProtectedDescendant) {
2226
- try {
2227
- await mergeBackupIntoLive(src, dst, carriedLivePaths);
2228
- } catch (err) {
2229
- failedCount += 1;
2230
- log.error(
2231
- { err, src, dst },
2232
- "Failed to merge backup subtree into live workspace during restore",
2233
- );
2234
- }
2235
- continue;
2236
- }
2237
-
2238
- // No carried descendants — safe to replace wholesale. If real already
2239
- // has this entry (partial swap-in), remove it first.
2240
- try {
2241
- await rm(dst, { recursive: true, force: true });
2242
- } catch (err) {
2243
- log.warn(
2244
- { err, dst },
2245
- "Failed to clear partial-swap entry before restoring from backup",
2246
- );
2247
- }
2248
- try {
2249
- await rename(src, dst);
2250
- } catch (err) {
2251
- if (isEXDEV(err)) {
2252
- try {
2253
- await copyTreeSkippingTransient(src, dst);
2254
- await rm(src, { recursive: true, force: true });
2255
- continue;
2256
- } catch (cpErr) {
2257
- failedCount += 1;
2258
- log.error(
2259
- { err: cpErr, src, dst },
2260
- "Failed to restore backup entry via cp fallback; manual recovery may be required",
2261
- );
2262
- continue;
2263
- }
2264
- }
2265
- failedCount += 1;
2266
- log.error(
2267
- { err, src, dst },
2268
- "Failed to restore backup entry; manual recovery may be required",
2269
- );
2270
- }
2271
- }
2272
-
2273
- return { ok: failedCount === 0, failedCount };
2274
- }
2275
-
2276
- /**
2277
- * Copy `src` (a backup subtree) into `dst` (the live-workspace subtree,
2278
- * which already exists and may already contain carried descendants we
2279
- * must not clobber). Each child in `src` that doesn't collide with an
2280
- * existing entry in `dst` is moved in; children that DO collide recurse
2281
- * so carried files deeper in the tree survive.
2282
- */
2283
- async function mergeBackupIntoLive(
2284
- src: string,
2285
- dst: string,
2286
- carriedLivePaths: readonly string[],
2287
- ): Promise<void> {
2288
- await mkdir(dst, { recursive: true });
2289
-
2290
- let children: string[];
2291
- try {
2292
- children = await readdir(src);
2293
- } catch (err) {
2294
- if (isENOENT(err)) {
2295
- return;
2296
- }
2297
- throw err;
2298
- }
2299
-
2300
- for (const childName of children) {
2301
- const childSrc = join(src, childName);
2302
- const childDst = join(dst, childName);
2303
- const childDstAbs = resolve(childDst);
2304
-
2305
- let dstExists = false;
2306
- try {
2307
- await stat(childDst);
2308
- dstExists = true;
2309
- } catch (err) {
2310
- if (!isENOENT(err)) {
2311
- throw err;
2312
- }
2313
- }
2314
-
2315
- if (!dstExists) {
2316
- try {
2317
- await rename(childSrc, childDst);
2318
- } catch (err) {
2319
- if (isEXDEV(err)) {
2320
- await copyTreeSkippingTransient(childSrc, childDst);
2321
- await rm(childSrc, { recursive: true, force: true });
2322
- } else {
2323
- throw err;
2324
- }
2325
- }
2326
- continue;
2327
- }
2328
-
2329
- // dst child exists — check whether it IS a carried entry or CONTAINS
2330
- // one. If it IS carried, backup's version is stale (carried is
2331
- // canonical). If it CONTAINS carried, recurse.
2332
- const isCarriedLeaf = carriedLivePaths.includes(childDstAbs);
2333
- if (isCarriedLeaf) {
2334
- // Skip — keep carried version that was already restored.
2335
- continue;
2336
- }
2337
- const containsCarried = carriedLivePaths.some(
2338
- (c) => c !== childDstAbs && c.startsWith(childDstAbs + sep),
2339
- );
2340
- if (containsCarried) {
2341
- await mergeBackupIntoLive(childSrc, childDst, carriedLivePaths);
2342
- continue;
2343
- }
2344
-
2345
- // No carried conflict — backup's version should win over whatever
2346
- // the partial-swap-in put here.
2347
- await rm(childDst, { recursive: true, force: true });
2348
- try {
2349
- await rename(childSrc, childDst);
2350
- } catch (err) {
2351
- if (isEXDEV(err)) {
2352
- await copyTreeSkippingTransient(childSrc, childDst);
2353
- await rm(childSrc, { recursive: true, force: true });
2354
- } else {
2355
- throw err;
2356
- }
2357
- }
2358
- }
2359
- }
2360
-
2361
- /**
2362
- * Resolve an archive path through the caller's resolver, then rebase the
2363
- * returned disk path onto the temp workspace. Returns `null` when the path
2364
- * cannot be resolved or lands outside `realWorkspaceDir`.
2365
- */
2366
- function resolveInsideTempWorkspace(
2367
- archivePath: string,
2368
- pathResolver: PathResolver,
2369
- realWorkspaceDir: string,
2370
- tempWorkspaceDir: string,
2371
- ): string | null {
2372
- const resolved = pathResolver.resolve(archivePath);
2373
- if (!resolved) {
2374
- return null;
2375
- }
2376
- return rebaseOntoTempWorkspace(resolved, realWorkspaceDir, tempWorkspaceDir);
2377
- }
2378
-
2379
- /**
2380
- * Replace the `realWorkspaceDir` prefix of `diskPath` with `tempWorkspaceDir`.
2381
- * Returns null if `diskPath` is not inside `realWorkspaceDir`.
2382
- */
2383
- function rebaseOntoTempWorkspace(
2384
- diskPath: string,
2385
- realWorkspaceDir: string,
2386
- tempWorkspaceDir: string,
2387
- ): string | null {
2388
- const resolved = resolve(diskPath);
2389
- const root = resolve(realWorkspaceDir);
2390
- if (resolved === root) {
2391
- return resolve(tempWorkspaceDir);
2392
- }
2393
- const prefix = root + sep;
2394
- if (!resolved.startsWith(prefix)) {
2395
- return null;
2396
- }
2397
- return resolve(tempWorkspaceDir, resolved.slice(prefix.length));
2398
- }
2399
-
2400
- /**
2401
- * Drain an entry body through the hash verifier, discarding the output.
2402
- *
2403
- * Uses `pipeline` (not `.pipe()`) so that if `body` is destroyed mid-stream
2404
- * — e.g. the upstream fetch body is torn down during a URL import — the
2405
- * verifier is destroyed too, and this call rejects promptly instead of
2406
- * hanging on a `for await` that never terminates.
2407
- *
2408
- * A `/dev/null` Writable sink terminates the chain so the verifier's
2409
- * readable side is continuously drained. Without this sink, a Transform as
2410
- * the last pipeline stage would stall once its internal buffer reached
2411
- * `highWaterMark` (16 KB default), since nothing would pull its output,
2412
- * and `pipeline` would hang indefinitely on any skipped entry >~16 KB.
2413
- */
2414
- async function drainThroughVerifier(
2415
- body: Readable,
2416
- expected: { sha256: string; size: number; archivePath: string },
2417
- ): Promise<void> {
2418
- const verifier = createHashVerifier(expected);
2419
- const devNull = new Writable({
2420
- write(_chunk, _enc, cb) {
2421
- cb();
2422
- },
2423
- });
2424
- await pipeline(body, verifier, devNull);
2425
- }
2426
-
2427
- /**
2428
- * Hard cap on the per-entry size that `collectHashVerified` is willing to
2429
- * buffer in memory. Applied to credential bodies and config files — both
2430
- * are expected to be KB-scale in practice. Exceeding this cap signals a
2431
- * crafted or corrupted bundle and is rejected before any bytes are read,
2432
- * so the streaming importer's memory guarantees still hold on a 3 GB pod
2433
- * even when the URL import is attacker-controlled.
2434
- */
2435
- const MAX_BUFFERED_ENTRY_BYTES = 16 * 1024 * 1024;
2436
-
2437
- /**
2438
- * Collect an entry body into a Buffer, verifying hash+size along the way.
2439
- *
2440
- * Uses `pipeline` + a sink writable that accumulates chunks, so destroy
2441
- * signals propagate the same way as `drainThroughVerifier` and the hash
2442
- * verifier's `_flush` (which asserts size+sha256) always runs.
2443
- *
2444
- * Rejects entries whose manifest-declared size exceeds
2445
- * `MAX_BUFFERED_ENTRY_BYTES` BEFORE reading any bytes, so an oversized
2446
- * credential or config file cannot drive RSS up by `expected.size` on a
2447
- * memory-limited pod.
2448
- */
2449
- async function collectHashVerified(
2450
- body: Readable,
2451
- expected: { sha256: string; size: number; archivePath: string },
2452
- ): Promise<Buffer> {
2453
- if (expected.size > MAX_BUFFERED_ENTRY_BYTES) {
2454
- body.destroy();
2455
- throw new StreamingValidationError(
2456
- "entry_too_large_to_buffer",
2457
- `Archive entry "${expected.archivePath}" declares ${expected.size} bytes, exceeding the ${MAX_BUFFERED_ENTRY_BYTES}-byte in-memory buffer cap for credentials/configs`,
2458
- expected.archivePath,
2459
- );
2460
- }
2461
- const verifier = createHashVerifier(expected);
2462
- const chunks: Buffer[] = [];
2463
- const sink = new Writable({
2464
- write(chunk, _enc, cb) {
2465
- chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk));
2466
- cb();
2467
- },
2468
- });
2469
- await pipeline(body, verifier, sink);
2470
- return Buffer.concat(chunks);
2471
- }
2472
-
2473
- /** Map a thrown error from streaming orchestration into an ImportCommitResult. */
2474
- function mapThrownToResult(err: unknown): ImportCommitResult {
2475
- if (err instanceof StreamingValidationError) {
2476
- return {
2477
- ok: false,
2478
- reason: "validation_failed",
2479
- errors: [
2480
- {
2481
- code: err.code,
2482
- message: err.message,
2483
- ...(err.archivePath !== undefined ? { path: err.archivePath } : {}),
2484
- },
2485
- ],
2486
- };
2487
- }
2488
-
2489
- if (err instanceof VersionIncompatibleError) {
2490
- return {
2491
- ok: false,
2492
- reason: "version_incompatible",
2493
- bundle_compat: err.bundleCompat,
2494
- runtime_version: err.runtimeVersion,
2495
- };
2496
- }
2497
-
2498
- // Errors we raised ourselves for disk-side failures.
2499
- if (err instanceof WriteFailedError) {
2500
- return {
2501
- ok: false,
2502
- reason: "write_failed",
2503
- message: err.message,
2504
- };
2505
- }
2506
-
2507
- // Anything else bubbling out of the tar / gunzip / HTTP stream pipeline:
2508
- // treat as extraction_failed. This matches the buffer-based validator's
2509
- // gzip/tar parse errors.
2510
- return {
2511
- ok: false,
2512
- reason: "extraction_failed",
2513
- message: errMessage(err),
2514
- };
2515
- }
2516
-
2517
- /** Sentinel error for disk I/O failures during streaming. */
2518
- class WriteFailedError extends Error {
2519
- constructor(message: string) {
2520
- super(message);
2521
- this.name = "WriteFailedError";
2522
- }
2523
- }
2524
-
2525
- function wrapWriteError(prefix: string, cause: unknown): WriteFailedError {
2526
- return new WriteFailedError(`${prefix}: ${errMessage(cause)}`);
2527
- }
2528
-
2529
- /**
2530
- * Sentinel error thrown when the bundle's manifest declares a runtime-version
2531
- * compat range that excludes the current `APP_VERSION`. Caught by the same
2532
- * try/catch that wraps the streaming parse loop so `cleanupTempDir()` runs
2533
- * before `mapThrownToResult` translates it into the `version_incompatible`
2534
- * shape of `ImportCommitResult`.
2535
- */
2536
- class VersionIncompatibleError extends Error {
2537
- constructor(
2538
- readonly bundleCompat: policy.RuntimeCompatibility,
2539
- readonly runtimeVersion: string,
2540
- ) {
2541
- super(
2542
- policy.formatRuntimeCompatibilityMessage(bundleCompat, runtimeVersion),
2543
- );
2544
- this.name = "VersionIncompatibleError";
2545
- }
2546
- }
2547
-
2548
- function errMessage(err: unknown): string {
2549
- return err instanceof Error ? err.message : String(err);
2550
- }
2551
-
2552
- function isENOENT(err: unknown): boolean {
2553
- return (
2554
- typeof err === "object" &&
2555
- err !== null &&
2556
- (err as { code?: string }).code === "ENOENT"
2557
- );
2558
- }
2559
-
2560
- function isEXDEV(err: unknown): boolean {
2561
- return (
2562
- typeof err === "object" &&
2563
- err !== null &&
2564
- (err as { code?: string }).code === "EXDEV"
2565
- );
2566
- }
2567
-
2568
- // ---------------------------------------------------------------------------
2569
- // Crash-recovery marker
2570
- //
2571
- // `streamCommitImport` moves preserved paths (SQLite DB, Qdrant, etc.) from
2572
- // the live workspace into a temp tree before the atomic rename pair. If the
2573
- // process is killed between those two phases the live workspace comes up
2574
- // missing the preserved paths. The marker written here persists the state
2575
- // needed to replay the recovery on the next start-up.
2576
- //
2577
- // Schema stays deliberately small so a partially-written marker is easy to
2578
- // detect (JSON parse failure → skip recovery rather than act on garbage).
2579
- // ---------------------------------------------------------------------------
2580
-
2581
- const ImportMarkerSchema = z.object({
2582
- /** Absolute path of the `.import-<uuid>` temp tree. */
2583
- tempWorkspaceDir: z.string(),
2584
- /** Preserved paths moved out of the live workspace pre-swap. */
2585
- carried: z.array(z.object({ liveChild: z.string(), tempChild: z.string() })),
2586
- /**
2587
- * Absolute path of the `${realWorkspaceDir}.pre-import-<ts>` backup dir
2588
- * (optional — only present once the content-level swap phase has started).
2589
- * `recoverInterruptedImport` moves entries from here back into
2590
- * `realWorkspaceDir` if it's populated, reversing any partial swap.
2591
- */
2592
- backupDir: z.string().optional(),
2593
- /**
2594
- * `true` once `swapWorkspaceContents` has returned successfully.
2595
- * `recoverInterruptedImport` checks this before restoring from
2596
- * `backupDir`: if the swap already completed, the backup is the OLD
2597
- * pre-import state and restoring it would silently undo the successful
2598
- * import. Instead, recovery just cleans up residual backup / temp
2599
- * artifacts.
2600
- */
2601
- swapCompleted: z.boolean().optional(),
2602
- });
2603
-
2604
- type ImportMarker = z.infer<typeof ImportMarkerSchema>;
2605
-
2606
- /** Basename of the recovery marker inside `realWorkspaceDir`. */
2607
- const IMPORT_MARKER_BASENAME = ".import-marker.json";
2608
-
2609
- /**
2610
- * Deterministic marker location INSIDE `realWorkspaceDir`.
2611
- *
2612
- * The marker must live on the same persistent volume as the scratch
2613
- * dirs (`.pre-import-<ts-uuid>`, `.import-<uuid>`). In Docker/Kubernetes
2614
- * the workspace is typically a mounted persistent volume while the
2615
- * container rootfs is ephemeral — a pod restart can drop files on
2616
- * rootfs while preserving the workspace, so a marker stored at
2617
- * `dirname(realWorkspaceDir)` could vanish across restart while the
2618
- * scratch dirs survive, leaving `recoverInterruptedImport` with
2619
- * nothing to act on and orphaning the interrupted state.
2620
- *
2621
- * The dot-prefix keeps it out of the way of normal content; phase 1 of
2622
- * `swapWorkspaceContents` filters it out via `scratchBasenames`, and
2623
- * the swap's content move also skips it so the marker stays in place
2624
- * across the workspace swap itself.
2625
- */
2626
- function importMarkerPathFor(realWorkspaceDir: string): string {
2627
- return join(realWorkspaceDir, IMPORT_MARKER_BASENAME);
2628
- }
2629
-
2630
- async function writeImportMarker(
2631
- markerPath: string,
2632
- marker: ImportMarker,
2633
- ): Promise<void> {
2634
- const serialized = JSON.stringify(marker);
2635
- const tmp = `${markerPath}.tmp-${randomUUID()}`;
2636
- // Write+rename so a crash mid-write leaves either the old marker (or
2637
- // nothing) rather than a truncated JSON blob.
2638
- await writeFile(tmp, serialized, { mode: 0o600 });
2639
- await rename(tmp, markerPath);
2640
- }
2641
-
2642
- async function safelyDeleteMarker(markerPath: string): Promise<void> {
2643
- try {
2644
- await unlink(markerPath);
2645
- } catch (err) {
2646
- if (isENOENT(err)) {
2647
- return;
2648
- }
2649
- log.warn({ err, markerPath }, "Failed to delete import-recovery marker");
2650
- }
2651
- }
2652
-
2653
- export interface RecoveryResult {
2654
- /**
2655
- * `true` when there's no leftover rollback state blocking a new
2656
- * import: no marker, successful restore, or a recorded
2657
- * `swapCompleted` fast-path cleanup. Callers (`streamCommitImport`,
2658
- * daemon start-up) can proceed safely.
2659
- *
2660
- * `false` when the rollback is incomplete — the marker / backup /
2661
- * temp tree are intentionally preserved on disk for a future retry,
2662
- * so any caller about to rewrite the marker must refuse to proceed
2663
- * to avoid orphaning the unresolved state.
2664
- */
2665
- ok: boolean;
2666
- /** Number of entries that couldn't be restored in the partial case. */
2667
- failedCount: number;
2668
- }
2669
-
2670
- /**
2671
- * Replay any crash-interrupted import against `realWorkspaceDir`.
2672
- *
2673
- * Call at daemon start-up (and implicitly at the start of every
2674
- * `streamCommitImport` as a self-healing belt) so a prior killed import
2675
- * doesn't leave the live workspace missing `data/db` / `data/qdrant` /
2676
- * `embedding-models` / `deprecated`.
2677
- *
2678
- * Best-effort: logs per-entry failures and keeps going rather than
2679
- * throwing. If no marker exists this is a cheap no-op. Returns a
2680
- * `RecoveryResult` so callers can distinguish "nothing to recover /
2681
- * recovered cleanly" from "rollback still pending — don't start
2682
- * anything new."
2683
- */
2684
- export async function recoverInterruptedImport(
2685
- realWorkspaceDir: string,
2686
- ): Promise<RecoveryResult> {
2687
- const markerPath = importMarkerPathFor(resolve(realWorkspaceDir));
2688
- let raw: string;
2689
- try {
2690
- raw = await readFile(markerPath, "utf8");
2691
- } catch (err) {
2692
- if (isENOENT(err)) {
2693
- return { ok: true, failedCount: 0 };
2694
- }
2695
- log.warn({ err, markerPath }, "Unable to read import-recovery marker");
2696
- return { ok: true, failedCount: 0 };
2697
- }
2698
-
2699
- let rawJson: unknown;
2700
- try {
2701
- rawJson = JSON.parse(raw);
2702
- } catch (err) {
2703
- log.warn(
2704
- { err, markerPath },
2705
- "Import-recovery marker is malformed; deleting without acting on it",
2706
- );
2707
- await safelyDeleteMarker(markerPath);
2708
- return { ok: true, failedCount: 0 };
2709
- }
2710
-
2711
- const parsed = ImportMarkerSchema.safeParse(rawJson);
2712
- if (!parsed.success) {
2713
- log.warn(
2714
- { markerPath, error: parsed.error.message },
2715
- "Import-recovery marker has unexpected shape; deleting",
2716
- );
2717
- await safelyDeleteMarker(markerPath);
2718
- return { ok: true, failedCount: 0 };
2719
- }
2720
- const marker = parsed.data;
2721
-
2722
- log.info(
2723
- {
2724
- markerPath,
2725
- tempWorkspaceDir: marker.tempWorkspaceDir,
2726
- carriedCount: marker.carried.length,
2727
- swapCompleted: marker.swapCompleted === true,
2728
- },
2729
- "Recovering from interrupted import",
2730
- );
2731
-
2732
- const carriedEntries = marker.carried.map((c) => ({
2733
- liveChild: c.liveChild,
2734
- tempChild: c.tempChild,
2735
- }));
2736
-
2737
- // FAST PATH: the previous process completed the swap but crashed before
2738
- // deleting the marker. Backup is the OLD pre-import state — restoring it
2739
- // would silently undo the successful import. Skip backup restore, skip
2740
- // carried restore (everything is already in live), just clean up
2741
- // artifacts.
2742
- if (marker.swapCompleted === true) {
2743
- if (typeof marker.backupDir === "string" && marker.backupDir.length > 0) {
2744
- await rm(marker.backupDir, { recursive: true, force: true }).catch(
2745
- (err) => {
2746
- log.warn(
2747
- { err, backupDir: marker.backupDir },
2748
- "Failed to clean up backup dir after completed import",
2749
- );
2750
- },
2751
- );
2752
- }
2753
- await rm(marker.tempWorkspaceDir, { recursive: true, force: true }).catch(
2754
- (err) => {
2755
- log.warn(
2756
- { err, tempWorkspaceDir: marker.tempWorkspaceDir },
2757
- "Failed to clean up temp workspace after completed import",
2758
- );
2759
- },
2760
- );
2761
- await safelyDeleteMarker(markerPath);
2762
- return { ok: true, failedCount: 0 };
2763
- }
2764
-
2765
- // SLOW PATH: swap did not complete. Roll back to pre-import state.
2766
- //
2767
- // Order matters: restore from backup FIRST, then restore carried
2768
- // entries. If carried ran first, a subsequent `restoreFromBackupDir`
2769
- // call that owns a parent dir (`data/`) would clobber the just-restored
2770
- // carried entries (`data/db`, `data/qdrant`). Backup-first + carrier-
2771
- // aware merge in `restoreFromBackupDir` preserves both.
2772
- let restoreResult: RestoreFromBackupResult = { ok: true, failedCount: 0 };
2773
- if (typeof marker.backupDir === "string" && marker.backupDir.length > 0) {
2774
- try {
2775
- restoreResult = await restoreFromBackupDir(
2776
- marker.backupDir,
2777
- resolve(realWorkspaceDir),
2778
- carriedEntries,
2779
- );
2780
- } catch (err) {
2781
- log.error(
2782
- { err, backupDir: marker.backupDir },
2783
- "Failed to restore from backup dir during import recovery; manual intervention may be required",
2784
- );
2785
- restoreResult = { ok: false, failedCount: 1 };
2786
- }
2787
- }
2788
-
2789
- await restoreCarriedPaths(carriedEntries);
2790
-
2791
- // Only drop the backup dir if the restore completed cleanly. A partial
2792
- // restore means there's still content in `backupDir` that no other
2793
- // state holds — keep it for manual / next-boot recovery.
2794
- if (
2795
- restoreResult.ok &&
2796
- typeof marker.backupDir === "string" &&
2797
- marker.backupDir.length > 0
2798
- ) {
2799
- await rm(marker.backupDir, { recursive: true, force: true }).catch(
2800
- (err) => {
2801
- log.warn(
2802
- { err, backupDir: marker.backupDir },
2803
- "Failed to clean up backup dir during import recovery",
2804
- );
2805
- },
2806
- );
2807
- } else if (!restoreResult.ok) {
2808
- log.error(
2809
- {
2810
- backupDir: marker.backupDir,
2811
- failedCount: restoreResult.failedCount,
2812
- },
2813
- "Backup restore had failures; preserving backup dir and marker for next-boot retry",
2814
- );
2815
- }
2816
-
2817
- // Only clean up the temp tree + marker when the restore completed
2818
- // cleanly. On a partial restore the marker's `carried` plan still
2819
- // references tempChild paths, so deleting the temp tree would break
2820
- // the next boot's recovery attempt — leave both in place for retry.
2821
- if (restoreResult.ok) {
2822
- try {
2823
- await rm(marker.tempWorkspaceDir, { recursive: true, force: true });
2824
- } catch (err) {
2825
- log.warn(
2826
- { err, tempWorkspaceDir: marker.tempWorkspaceDir },
2827
- "Failed to clean up temp workspace during import recovery",
2828
- );
2829
- }
2830
- await safelyDeleteMarker(markerPath);
2831
- } else {
2832
- log.warn(
2833
- {
2834
- tempWorkspaceDir: marker.tempWorkspaceDir,
2835
- markerPath,
2836
- },
2837
- "Preserving temp tree + marker for next-boot recovery retry",
2838
- );
2839
- }
2840
-
2841
- return restoreResult.ok
2842
- ? { ok: true, failedCount: 0 }
2843
- : { ok: false, failedCount: restoreResult.failedCount };
2844
- }