@cruxy/cli 1.11.1 → 1.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checkpoint/service.js +44 -3
- package/dist/cli/commands/sessions.js +8 -0
- package/dist/errors/constructors.js +15 -6
- package/dist/errors/types.js +7 -0
- package/dist/indexing/embedder.js +34 -11
- package/dist/indexing/model-cache.js +399 -0
- package/dist/session/index.js +1 -0
- package/dist/session/log.js +66 -0
- package/dist/session/owner.js +123 -0
- package/dist/session/prune.js +11 -0
- package/dist/session/resume.js +22 -3
- package/dist/subagent/orchestrator.js +2 -2
- package/dist/subagent/registry-scope.js +28 -5
- package/dist/tools/file/apply-patch.js +49 -23
- package/dist/tools/file/edit-file.js +15 -1
- package/dist/tools/file/snapshot.js +63 -0
- package/dist/tools/file/write-file.js +26 -5
- package/dist/utils/process-owner.js +107 -0
- package/package.json +3 -2
|
@@ -8,6 +8,7 @@ import { captureFiles } from "./capture.js";
|
|
|
8
8
|
import { GitCheckpointStore } from "./git-store.js";
|
|
9
9
|
import { ShadowCheckpointStore } from "./shadow-store.js";
|
|
10
10
|
import { applyRollback, buildRollbackPreview, computeRollbackPlan, } from "./restore.js";
|
|
11
|
+
import { describeOwner, selfStamp, } from "../utils/process-owner.js";
|
|
11
12
|
/** Is `root` inside a git working tree? (Decides the checkpoint substrate.) */
|
|
12
13
|
export function isGitWorkTree(root) {
|
|
13
14
|
const res = runGitCapture(["rev-parse", "--is-inside-work-tree"], root);
|
|
@@ -112,6 +113,7 @@ export class CheckpointService {
|
|
|
112
113
|
files: entries,
|
|
113
114
|
touchedPaths: [],
|
|
114
115
|
hasShellMutations: false,
|
|
116
|
+
owner: selfStamp(),
|
|
115
117
|
};
|
|
116
118
|
await this.writeManifest(checkpoint);
|
|
117
119
|
await this.prune();
|
|
@@ -203,10 +205,27 @@ export class CheckpointService {
|
|
|
203
205
|
const applied = await applyRollback(this.root, plan, store);
|
|
204
206
|
return { kind: "applied", checkpoint, applied };
|
|
205
207
|
}
|
|
206
|
-
/**
|
|
208
|
+
/**
|
|
209
|
+
* Enforce `checkpoint.retention`: drop oldest manifests, then GC content.
|
|
210
|
+
*
|
|
211
|
+
* ACROSS PROCESSES (P1). `.cruxy/checkpoints/` is per root, not per process,
|
|
212
|
+
* and every cruxy in the project prunes it by count — so a second session
|
|
213
|
+
* creating its own checkpoints pushed the first session's out of the
|
|
214
|
+
* window and swept their objects, and the first session's `cruxy rollback`
|
|
215
|
+
* then failed or restored the wrong thing. A checkpoint whose recorded
|
|
216
|
+
* owner is another LIVE process is therefore not this process's to prune:
|
|
217
|
+
* it is set aside, its objects stay referenced, and the retention count is
|
|
218
|
+
* applied to everything else. The bound is exceeded by at most the live
|
|
219
|
+
* foreign checkpoints, and only while their owners run: a crashed owner's
|
|
220
|
+
* checkpoints are ordinary candidates the next time anyone prunes, because
|
|
221
|
+
* liveness is pid + start-time asked of the OS, never a file that must be
|
|
222
|
+
* cleaned up.
|
|
223
|
+
*/
|
|
207
224
|
async prune() {
|
|
208
225
|
const all = await this.list();
|
|
209
|
-
const
|
|
226
|
+
const protectedIds = foreignLive(all);
|
|
227
|
+
const candidates = all.filter((c) => !protectedIds.has(c.id));
|
|
228
|
+
const doomed = candidates.slice(this.config.checkpoint.retention);
|
|
210
229
|
if (doomed.length === 0)
|
|
211
230
|
return;
|
|
212
231
|
for (const checkpoint of doomed) {
|
|
@@ -214,7 +233,7 @@ export class CheckpointService {
|
|
|
214
233
|
force: true,
|
|
215
234
|
});
|
|
216
235
|
}
|
|
217
|
-
const survivors = all.
|
|
236
|
+
const survivors = all.filter((c) => !doomed.includes(c));
|
|
218
237
|
const referenced = new Set(survivors.flatMap((c) => c.files.map((f) => f.oid)));
|
|
219
238
|
// The shadow pool is ours to sweep; git's dangling objects belong to git gc.
|
|
220
239
|
await new ShadowCheckpointStore(this.root).collect(referenced);
|
|
@@ -266,6 +285,28 @@ export class CheckpointService {
|
|
|
266
285
|
return parsed;
|
|
267
286
|
}
|
|
268
287
|
}
|
|
288
|
+
/**
|
|
289
|
+
* Ids of the checkpoints another LIVE process owns (P1 — see {@link prune}).
|
|
290
|
+
* One liveness lookup per distinct owner, not per manifest: the darwin lookup
|
|
291
|
+
* is a `ps` call, and a project can hold many checkpoints from one session.
|
|
292
|
+
*/
|
|
293
|
+
function foreignLive(all) {
|
|
294
|
+
const ids = new Set();
|
|
295
|
+
const status = new Map();
|
|
296
|
+
for (const c of all) {
|
|
297
|
+
if (!c.owner)
|
|
298
|
+
continue;
|
|
299
|
+
const key = `${c.owner.pid}:${c.owner.token}`;
|
|
300
|
+
let s = status.get(key);
|
|
301
|
+
if (s === undefined) {
|
|
302
|
+
s = describeOwner(c.owner);
|
|
303
|
+
status.set(key, s);
|
|
304
|
+
}
|
|
305
|
+
if (s === "live")
|
|
306
|
+
ids.add(c.id);
|
|
307
|
+
}
|
|
308
|
+
return ids;
|
|
309
|
+
}
|
|
269
310
|
/** `ck-<utc-stamp>-<rand>` — sortable, collision-safe enough for a local CLI. */
|
|
270
311
|
function newCheckpointId() {
|
|
271
312
|
const stamp = new Date()
|
|
@@ -7,6 +7,7 @@ import { SESSION_FILE_EXT, matchSessionRefs, listSessionRefs, pruneSessions, ses
|
|
|
7
7
|
import { themeForColor } from "../../theme/index.js";
|
|
8
8
|
import { formatBytes } from "../../utils/disk.js";
|
|
9
9
|
import { logger } from "../../utils/logger.js";
|
|
10
|
+
import { describeHolder, removeOwnerFile, sessionHeldBy, } from "../../session/owner.js";
|
|
10
11
|
/**
|
|
11
12
|
* `cruxy sessions` (#257) — see and bound what `~/.cruxy/projects/<project>/`
|
|
12
13
|
* is holding.
|
|
@@ -149,8 +150,15 @@ export function sessionsCommand() {
|
|
|
149
150
|
"sessions are per-directory; check you are in the right one",
|
|
150
151
|
]);
|
|
151
152
|
}
|
|
153
|
+
// Not while another cruxy is writing it (P1): its next append would
|
|
154
|
+
// recreate the file headless and leave a session nothing can load.
|
|
155
|
+
const holder = sessionHeldBy(matches[0].file);
|
|
156
|
+
if (holder) {
|
|
157
|
+
throw usageError(`session ${shortId(matches[0].sessionId)} is open in another cruxy (${describeHolder(holder)})`, ["quit that cruxy first, then delete it"]);
|
|
158
|
+
}
|
|
152
159
|
try {
|
|
153
160
|
unlinkSync(matches[0].file);
|
|
161
|
+
removeOwnerFile(matches[0].file);
|
|
154
162
|
}
|
|
155
163
|
catch (err) {
|
|
156
164
|
throw usageError(`could not delete session ${shortId(matches[0].sessionId)}`, [`${matches[0].file}: ${err.message}`]);
|
|
@@ -545,16 +545,25 @@ export function indexEmbedderUnavailable(underlying) {
|
|
|
545
545
|
* exact "reads like success, isn't" trap C.17 forbids. The `search_codebase`
|
|
546
546
|
* tool instead surfaces it as a tool error and points the model at `grep_files`.
|
|
547
547
|
*/
|
|
548
|
-
export function indexEmbedderDownloadFailed(underlying) {
|
|
548
|
+
export function indexEmbedderDownloadFailed(underlying, opts = {}) {
|
|
549
|
+
// A refused archive is not a connectivity problem: the download succeeded
|
|
550
|
+
// and cruxy's extractor rejected an entry (path traversal, a link, an
|
|
551
|
+
// unexpected layout). Retrying will refuse it again, so say so.
|
|
552
|
+
const nextSteps = opts.archiveRefused
|
|
553
|
+
? [
|
|
554
|
+
"the model archive was refused by cruxy's extraction guard and nothing was written; this is not a connectivity problem",
|
|
555
|
+
"do not retry blindly — if it persists, report it with the message above (the upstream archive may have changed shape)",
|
|
556
|
+
]
|
|
557
|
+
: [
|
|
558
|
+
"check your internet connection — the model (bge-small-en-v1.5, ~77 MB) downloads once on first use",
|
|
559
|
+
"if you are behind a proxy or firewall, allow access to storage.googleapis.com (the qdrant-fastembed bucket); HTTPS_PROXY / NO_PROXY are honored",
|
|
560
|
+
"once the download succeeds it is cached under ~/.cruxy/models and never re-fetched",
|
|
561
|
+
];
|
|
549
562
|
return new CruxyError({
|
|
550
563
|
code: ErrorCode.IndexEmbedderDownloadFailed,
|
|
551
564
|
title: "the local embedding model could not be downloaded or initialized",
|
|
552
565
|
cause: messageOf(underlying),
|
|
553
|
-
nextSteps
|
|
554
|
-
"check your internet connection — the model (bge-small-en-v1.5) downloads once on first use",
|
|
555
|
-
"if you are behind a proxy or firewall, allow access to the model host (Hugging Face) and set HTTPS_PROXY",
|
|
556
|
-
"once the download succeeds it is cached under ~/.cruxy/models and never re-fetched",
|
|
557
|
-
],
|
|
566
|
+
nextSteps,
|
|
558
567
|
underlying,
|
|
559
568
|
});
|
|
560
569
|
}
|
package/dist/errors/types.js
CHANGED
|
@@ -99,6 +99,12 @@ export const ErrorCode = {
|
|
|
99
99
|
PermissionDenied: "CRUXY_E_PERMISSION_DENIED",
|
|
100
100
|
PathEscape: "CRUXY_E_PATH_ESCAPE",
|
|
101
101
|
CheckpointFailed: "CRUXY_E_CHECKPOINT_FAILED",
|
|
102
|
+
/** A mutating file tool re-read its target immediately before the approved
|
|
103
|
+
* write and found it was not the file the approval was granted against (P1):
|
|
104
|
+
* changed, deleted, or created by something else during the approval wait.
|
|
105
|
+
* Refused with nothing written — the approval covered a diff against ONE
|
|
106
|
+
* specific state, and that state moved. See `tools/file/snapshot.ts`. */
|
|
107
|
+
FileChangedSinceRead: "CRUXY_E_FILE_CHANGED_SINCE_READ",
|
|
102
108
|
// index (exit 8)
|
|
103
109
|
/** The fastembed native module could not be LOADED (missing/broken install,
|
|
104
110
|
* un-built onnxruntime-node addon). Fail-loud by design — the embedder never
|
|
@@ -341,6 +347,7 @@ const EXIT_CODES = {
|
|
|
341
347
|
[ErrorCode.PermissionDenied]: 7,
|
|
342
348
|
[ErrorCode.PathEscape]: 7,
|
|
343
349
|
[ErrorCode.CheckpointFailed]: 7,
|
|
350
|
+
[ErrorCode.FileChangedSinceRead]: 7,
|
|
344
351
|
[ErrorCode.IndexEmbedderUnavailable]: 8,
|
|
345
352
|
[ErrorCode.IndexEmbedderDownloadFailed]: 8,
|
|
346
353
|
[ErrorCode.IndexStoreUnavailable]: 8,
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { globalDir } from "../config/paths.js";
|
|
2
3
|
import { indexEmbedderDownloadFailed, indexEmbedderUnavailable, } from "../errors/index.js";
|
|
4
|
+
import { ensureModelDir, MODEL_ONNX_FILE, } from "./model-cache.js";
|
|
3
5
|
import { l2normalize } from "./util.js";
|
|
4
6
|
/**
|
|
5
7
|
* Output dimensionality of bge-small-en-v1.5, and the default size of the
|
|
@@ -69,6 +71,12 @@ export class HashingEmbedder {
|
|
|
69
71
|
* registering the `search_codebase` tool stays cheap and the heavy ONNX runtime
|
|
70
72
|
* only loads when an index is actually built or queried.
|
|
71
73
|
*
|
|
74
|
+
* The model files are provisioned by cruxy ({@link ensureModelDir}), NOT by
|
|
75
|
+
* fastembed: fastembed is initialised with `model: CUSTOM` and an absolute
|
|
76
|
+
* directory, which in its `init` makes its own `retrieveModel` (tar@6-based
|
|
77
|
+
* download + extract, #306) unreachable. `CUSTOM` changes nothing else for this
|
|
78
|
+
* model — fastembed's only model-specific branch is for multilingual-e5.
|
|
79
|
+
*
|
|
72
80
|
* Embedding is CPU-bound and single-threaded inside ONNX, so throughput is
|
|
73
81
|
* bounded by `batchSize` (fed sequentially through fastembed's batching
|
|
74
82
|
* generator) rather than by JS-level concurrency.
|
|
@@ -84,23 +92,29 @@ export class FastEmbedEmbedder {
|
|
|
84
92
|
getModel() {
|
|
85
93
|
if (!this.model) {
|
|
86
94
|
this.model = (async () => {
|
|
95
|
+
const cacheDir = this.opts.cacheDir ?? path.join(globalDir(), "models");
|
|
96
|
+
const provision = this.opts.provision ?? ((dir) => ensureModelDir({ cacheDir: dir }));
|
|
97
|
+
let modelDir;
|
|
98
|
+
try {
|
|
99
|
+
modelDir = await provision(cacheDir);
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
throw indexEmbedderDownloadFailed(err, {
|
|
103
|
+
archiveRefused: isRefusedArchive(err),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
87
106
|
try {
|
|
88
107
|
const mod = await import("fastembed");
|
|
89
|
-
// fastembed's init does a non-recursive mkdir of the cache dir, so it
|
|
90
|
-
// fails if an ancestor (e.g. ~/.cruxy) doesn't exist yet. Create it first.
|
|
91
|
-
if (this.opts.cacheDir) {
|
|
92
|
-
await fs.mkdir(this.opts.cacheDir, { recursive: true });
|
|
93
|
-
}
|
|
94
108
|
return (await mod.FlagEmbedding.init({
|
|
95
|
-
model: mod.EmbeddingModel.
|
|
109
|
+
model: mod.EmbeddingModel.CUSTOM,
|
|
110
|
+
modelAbsoluteDirPath: modelDir,
|
|
111
|
+
modelName: MODEL_ONNX_FILE,
|
|
96
112
|
maxLength: this.opts.maxLength ?? 512,
|
|
97
|
-
|
|
98
|
-
showDownloadProgress: this.opts.showDownloadProgress ?? false,
|
|
113
|
+
showDownloadProgress: false,
|
|
99
114
|
}));
|
|
100
115
|
}
|
|
101
116
|
catch (err) {
|
|
102
|
-
//
|
|
103
|
-
// init failed (offline, unreachable model bucket, proxy). Surface a
|
|
117
|
+
// ONNX-runtime init failed on a verified model directory. Surface a
|
|
104
118
|
// typed, actionable error instead of letting it collapse into the
|
|
105
119
|
// generic CRUXY_E_INDEX_FAILED ("re-run --verbose"). Still fail-loud —
|
|
106
120
|
// this never degrades to the lexical backend. (Module-*load* failure is
|
|
@@ -128,6 +142,15 @@ export class FastEmbedEmbedder {
|
|
|
128
142
|
return l2normalize(Float32Array.from(await model.queryEmbed(text)));
|
|
129
143
|
}
|
|
130
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Duck-typed on purpose: tests re-import this module across
|
|
147
|
+
* `vi.resetModules()`, which would defeat an `instanceof ModelCacheError`.
|
|
148
|
+
*/
|
|
149
|
+
function isRefusedArchive(err) {
|
|
150
|
+
return (err instanceof Error &&
|
|
151
|
+
err.name === "ModelCacheError" &&
|
|
152
|
+
err.kind === "refused");
|
|
153
|
+
}
|
|
131
154
|
/**
|
|
132
155
|
* Build the **production** embedder: fastembed / bge-small-en-v1.5.
|
|
133
156
|
*
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { EnvHttpProxyAgent, fetch as undiciFetch } from "undici";
|
|
5
|
+
import { extract as tarExtract } from "tar";
|
|
6
|
+
/**
|
|
7
|
+
* Cruxy-owned download + extraction of the local embedding model.
|
|
8
|
+
*
|
|
9
|
+
* Why this exists (#306): fastembed's own `retrieveModel` extracts the model
|
|
10
|
+
* tarball with `tar@6.2.1`, which carries a family of path-traversal /
|
|
11
|
+
* symlink-poisoning advisories, has no fixed 6.x, and cannot be overridden to
|
|
12
|
+
* tar 7 (fastembed does `import tar from "tar"`; tar 7 has no default export —
|
|
13
|
+
* the override was tested and breaks `cruxy index` at module link). fastembed
|
|
14
|
+
* 2.1.0 is its latest release and pins `tar ^6.2.0`.
|
|
15
|
+
*
|
|
16
|
+
* So cruxy provisions the model directory itself and hands fastembed an
|
|
17
|
+
* absolute path via `model: CUSTOM` + `modelAbsoluteDirPath`. In fastembed's
|
|
18
|
+
* `init`, `retrieveModel` sits on the other arm of a ternary on `CUSTOM`, so it
|
|
19
|
+
* is unreachable once we go through this module — tar 6 stays in the
|
|
20
|
+
* dependency tree (scanners will keep flagging it) but never runs.
|
|
21
|
+
*
|
|
22
|
+
* This code writes to every user's disk, so the archive is treated as hostile:
|
|
23
|
+
* - every entry is checked by an explicit allow-list (regular files and
|
|
24
|
+
* directories only; a single top-level directory named for the model; no
|
|
25
|
+
* `..`, absolute, drive-letter, backslash or NUL paths; bounded count and
|
|
26
|
+
* size) — never the library's defaults alone. A refused entry fails the
|
|
27
|
+
* whole extraction; it is never silently skipped.
|
|
28
|
+
* - download and extraction happen in a staging directory next to the final
|
|
29
|
+
* one; the final directory appears only via a rename after the required
|
|
30
|
+
* file set has been verified, so an interrupted run can never leave a
|
|
31
|
+
* directory that looks complete.
|
|
32
|
+
* - a completion marker records the verified file sizes; a cache hit requires
|
|
33
|
+
* the marker AND matching sizes, so a partial or tampered directory is
|
|
34
|
+
* redone rather than used.
|
|
35
|
+
*/
|
|
36
|
+
/** Model directory name inside the cache; also the archive's top-level dir. */
|
|
37
|
+
export const MODEL_NAME = "fast-bge-small-en-v1.5";
|
|
38
|
+
/** Where fastembed (and Qdrant's Python fastembed) fetch this archive from. */
|
|
39
|
+
export const MODEL_ARCHIVE_URL = `https://storage.googleapis.com/qdrant-fastembed/${MODEL_NAME}.tar.gz`;
|
|
40
|
+
/** The ONNX graph fastembed loads from the model directory. */
|
|
41
|
+
export const MODEL_ONNX_FILE = "model_optimized.onnx";
|
|
42
|
+
/**
|
|
43
|
+
* Files fastembed's `init` reads. The archive also ships `ort_config.json` and
|
|
44
|
+
* `vocab.txt`; they are extracted if present but not required.
|
|
45
|
+
*/
|
|
46
|
+
export const REQUIRED_FILES = [
|
|
47
|
+
"config.json",
|
|
48
|
+
MODEL_ONNX_FILE,
|
|
49
|
+
"special_tokens_map.json",
|
|
50
|
+
"tokenizer.json",
|
|
51
|
+
"tokenizer_config.json",
|
|
52
|
+
];
|
|
53
|
+
/** Written into the model dir after verification; absent ⇒ incomplete. */
|
|
54
|
+
export const MARKER_FILE = ".cruxy-model.json";
|
|
55
|
+
/** Compressed archive ceiling (the real one is ~77 MB). */
|
|
56
|
+
export const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024;
|
|
57
|
+
/** Sum of declared entry sizes ceiling (the real one is ~134 MB). */
|
|
58
|
+
export const MAX_EXTRACTED_BYTES = 1024 * 1024 * 1024;
|
|
59
|
+
/** Entry-count ceiling (the real archive has 8). */
|
|
60
|
+
export const MAX_ENTRIES = 64;
|
|
61
|
+
/** Abort the download if no bytes arrive for this long. */
|
|
62
|
+
const IDLE_TIMEOUT_MS = 60_000;
|
|
63
|
+
/** Staging dirs older than this are leftovers of a dead run and are swept. */
|
|
64
|
+
const STALE_STAGING_MS = 6 * 60 * 60 * 1000;
|
|
65
|
+
/**
|
|
66
|
+
* Failure of the download / extraction / verification pipeline. `kind` lets
|
|
67
|
+
* the caller pick next steps: a `download` failure is the classic offline
|
|
68
|
+
* first run; `refused` means the archive contained an entry the allow-list
|
|
69
|
+
* rejected (retrying blindly is wrong); `incomplete` means the archive did not
|
|
70
|
+
* contain the required file set, or the extracted set failed verification.
|
|
71
|
+
*/
|
|
72
|
+
export class ModelCacheError extends Error {
|
|
73
|
+
kind;
|
|
74
|
+
constructor(kind, message, cause) {
|
|
75
|
+
super(message, cause === undefined ? undefined : { cause });
|
|
76
|
+
this.name = "ModelCacheError";
|
|
77
|
+
this.kind = kind;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Ensure `<cacheDir>/<MODEL_NAME>` holds a verified, complete model and return
|
|
82
|
+
* that absolute path. Downloads and extracts only when the cache is missing or
|
|
83
|
+
* fails verification; never trusts a directory without its marker.
|
|
84
|
+
*/
|
|
85
|
+
export async function ensureModelDir(opts) {
|
|
86
|
+
const cacheDir = path.resolve(opts.cacheDir);
|
|
87
|
+
const modelDir = path.join(cacheDir, MODEL_NAME);
|
|
88
|
+
if (await isCompleteModelDir(modelDir))
|
|
89
|
+
return modelDir;
|
|
90
|
+
await fs.mkdir(cacheDir, { recursive: true });
|
|
91
|
+
await sweepStaleStaging(cacheDir);
|
|
92
|
+
// Stage next to the destination so the final publish is a same-filesystem
|
|
93
|
+
// rename (atomic on POSIX; on Windows it either succeeds or throws).
|
|
94
|
+
const stagingDir = path.join(cacheDir, `.${MODEL_NAME}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`);
|
|
95
|
+
const archivePath = `${stagingDir}.tar.gz`;
|
|
96
|
+
try {
|
|
97
|
+
await fs.mkdir(stagingDir, { recursive: true });
|
|
98
|
+
await downloadArchive(opts.url ?? MODEL_ARCHIVE_URL, archivePath, opts.fetchImpl, opts.onProgress);
|
|
99
|
+
opts.onProgress?.({ phase: "extract" });
|
|
100
|
+
await extractArchive(archivePath, stagingDir);
|
|
101
|
+
opts.onProgress?.({ phase: "verify" });
|
|
102
|
+
const extracted = path.join(stagingDir, MODEL_NAME);
|
|
103
|
+
const files = await verifyModelDir(extracted);
|
|
104
|
+
await writeMarker(extracted, files);
|
|
105
|
+
await publish(extracted, modelDir);
|
|
106
|
+
return modelDir;
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
await fs.rm(archivePath, { force: true }).catch(() => { });
|
|
110
|
+
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => { });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* True only when the marker exists, parses, names this model, and every
|
|
115
|
+
* required file is a regular file whose size matches the marker. Anything
|
|
116
|
+
* else — a missing marker (a partial or legacy extraction), a size mismatch,
|
|
117
|
+
* a symlink where a file should be — is "incomplete" and gets redone.
|
|
118
|
+
*/
|
|
119
|
+
export async function isCompleteModelDir(modelDir) {
|
|
120
|
+
let marker;
|
|
121
|
+
try {
|
|
122
|
+
const raw = await fs.readFile(path.join(modelDir, MARKER_FILE), "utf8");
|
|
123
|
+
marker = JSON.parse(raw);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return false;
|
|
127
|
+
}
|
|
128
|
+
if (!marker ||
|
|
129
|
+
marker.version !== 1 ||
|
|
130
|
+
marker.model !== MODEL_NAME ||
|
|
131
|
+
typeof marker.files !== "object" ||
|
|
132
|
+
marker.files === null) {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
for (const name of REQUIRED_FILES) {
|
|
136
|
+
const expected = marker.files[name];
|
|
137
|
+
if (typeof expected !== "number" || expected <= 0)
|
|
138
|
+
return false;
|
|
139
|
+
const st = await fs.lstat(path.join(modelDir, name)).catch(() => null);
|
|
140
|
+
if (!st || !st.isFile() || st.size !== expected)
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
// ── download ──────────────────────────────────────────────────────────────────
|
|
146
|
+
async function downloadArchive(url, dest, fetchImpl, onProgress) {
|
|
147
|
+
const doFetch = fetchImpl ?? undiciFetch;
|
|
148
|
+
const controller = new AbortController();
|
|
149
|
+
let idle;
|
|
150
|
+
const armIdle = () => {
|
|
151
|
+
if (idle)
|
|
152
|
+
clearTimeout(idle);
|
|
153
|
+
idle = setTimeout(() => controller.abort(new Error(`no data for ${IDLE_TIMEOUT_MS}ms`)), IDLE_TIMEOUT_MS);
|
|
154
|
+
idle.unref?.();
|
|
155
|
+
};
|
|
156
|
+
let res;
|
|
157
|
+
try {
|
|
158
|
+
armIdle();
|
|
159
|
+
res = await doFetch(url, {
|
|
160
|
+
signal: controller.signal,
|
|
161
|
+
redirect: "follow",
|
|
162
|
+
// Honor HTTPS_PROXY / NO_PROXY for this fixed, public URL (the web tool
|
|
163
|
+
// deliberately does not; it has an SSRF guard to protect instead). The
|
|
164
|
+
// agent is built only when a proxy is configured: undici prints an
|
|
165
|
+
// "experimental" warning on construction, and non-proxy users should
|
|
166
|
+
// never see it.
|
|
167
|
+
...(fetchImpl || !proxyConfigured()
|
|
168
|
+
? {}
|
|
169
|
+
: { dispatcher: new EnvHttpProxyAgent() }),
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
if (idle)
|
|
174
|
+
clearTimeout(idle);
|
|
175
|
+
throw new ModelCacheError("download", `downloading ${url} failed: ${messageOf(err)}`, err);
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
if (!res.ok) {
|
|
179
|
+
throw new ModelCacheError("download", `downloading ${url} failed: HTTP ${res.status}`);
|
|
180
|
+
}
|
|
181
|
+
const declared = Number(res.headers.get("content-length") ?? "");
|
|
182
|
+
if (Number.isFinite(declared) && declared > MAX_ARCHIVE_BYTES) {
|
|
183
|
+
throw new ModelCacheError("refused", `refused model archive: Content-Length ${declared} exceeds the ${MAX_ARCHIVE_BYTES}-byte ceiling`);
|
|
184
|
+
}
|
|
185
|
+
if (!res.body) {
|
|
186
|
+
throw new ModelCacheError("download", `downloading ${url}: empty body`);
|
|
187
|
+
}
|
|
188
|
+
const total = Number.isFinite(declared) && declared > 0 ? declared : undefined;
|
|
189
|
+
const fh = await fs.open(dest, "w");
|
|
190
|
+
let received = 0;
|
|
191
|
+
try {
|
|
192
|
+
for await (const chunk of res.body) {
|
|
193
|
+
armIdle();
|
|
194
|
+
received += chunk.byteLength;
|
|
195
|
+
if (received > MAX_ARCHIVE_BYTES) {
|
|
196
|
+
controller.abort();
|
|
197
|
+
throw new ModelCacheError("refused", `refused model archive: body exceeds the ${MAX_ARCHIVE_BYTES}-byte ceiling`);
|
|
198
|
+
}
|
|
199
|
+
await fh.write(chunk);
|
|
200
|
+
onProgress?.({ phase: "download", received, total });
|
|
201
|
+
}
|
|
202
|
+
if (total !== undefined && received !== total) {
|
|
203
|
+
throw new ModelCacheError("download", `downloading ${url}: connection closed after ${received} of ${total} bytes`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
if (err instanceof ModelCacheError)
|
|
208
|
+
throw err;
|
|
209
|
+
throw new ModelCacheError("download", `downloading ${url} failed: ${messageOf(err)}`, err);
|
|
210
|
+
}
|
|
211
|
+
finally {
|
|
212
|
+
await fh.close();
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
finally {
|
|
216
|
+
if (idle)
|
|
217
|
+
clearTimeout(idle);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// ── extraction ────────────────────────────────────────────────────────────────
|
|
221
|
+
/**
|
|
222
|
+
* Decide whether one archive entry may be written. Returns a reason string to
|
|
223
|
+
* refuse, or null to allow. Deliberately an allow-list: anything not proven
|
|
224
|
+
* safe is refused. Runs BEFORE tar's own checks, and independently of them.
|
|
225
|
+
*/
|
|
226
|
+
export function refuseEntry(entryPath, type, linkpath) {
|
|
227
|
+
if (type !== "File" && type !== "Directory") {
|
|
228
|
+
return `entry type ${type} is not allowed (${entryPath})`;
|
|
229
|
+
}
|
|
230
|
+
if (linkpath)
|
|
231
|
+
return `entry carries a link target (${entryPath})`;
|
|
232
|
+
if (entryPath.length === 0)
|
|
233
|
+
return "empty entry path";
|
|
234
|
+
if (entryPath.includes("\0"))
|
|
235
|
+
return "entry path contains NUL";
|
|
236
|
+
if (entryPath.includes("\\"))
|
|
237
|
+
return `entry path contains a backslash (${entryPath})`;
|
|
238
|
+
if (entryPath.startsWith("/"))
|
|
239
|
+
return `absolute entry path (${entryPath})`;
|
|
240
|
+
if (/^[A-Za-z]:/.test(entryPath))
|
|
241
|
+
return `drive-letter entry path (${entryPath})`;
|
|
242
|
+
const parts = entryPath.split("/");
|
|
243
|
+
// A directory entry may end with "/"; nothing else may have an empty segment.
|
|
244
|
+
if (type === "Directory" &&
|
|
245
|
+
parts.length > 1 &&
|
|
246
|
+
parts[parts.length - 1] === "") {
|
|
247
|
+
parts.pop();
|
|
248
|
+
}
|
|
249
|
+
for (const seg of parts) {
|
|
250
|
+
if (seg === "" || seg === "." || seg === "..") {
|
|
251
|
+
return `entry path has an empty, "." or ".." segment (${entryPath})`;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (parts[0] !== MODEL_NAME) {
|
|
255
|
+
return `entry is outside the ${MODEL_NAME}/ directory (${entryPath})`;
|
|
256
|
+
}
|
|
257
|
+
if (type === "Directory" && parts.length !== 1) {
|
|
258
|
+
return `nested directory not allowed (${entryPath})`;
|
|
259
|
+
}
|
|
260
|
+
if (type === "File" && parts.length !== 2) {
|
|
261
|
+
return `file must sit directly under ${MODEL_NAME}/ (${entryPath})`;
|
|
262
|
+
}
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
async function extractArchive(archivePath, cwd) {
|
|
266
|
+
const refused = [];
|
|
267
|
+
let entries = 0;
|
|
268
|
+
let declaredBytes = 0;
|
|
269
|
+
// On the extract path tar always hands a ReadEntry (Stats is the create-side
|
|
270
|
+
// shape of the same option), but keep the narrowing explicit.
|
|
271
|
+
const filter = (entryPath, entry) => {
|
|
272
|
+
entries += 1;
|
|
273
|
+
if (entries > MAX_ENTRIES) {
|
|
274
|
+
refused.push(`more than ${MAX_ENTRIES} entries`);
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
if (!("header" in entry)) {
|
|
278
|
+
refused.push(`unexpected non-archive entry (${entryPath})`);
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
const reason = refuseEntry(entryPath, String(entry.type), entry.linkpath ? String(entry.linkpath) : undefined);
|
|
282
|
+
if (reason) {
|
|
283
|
+
refused.push(reason);
|
|
284
|
+
return false;
|
|
285
|
+
}
|
|
286
|
+
declaredBytes += Number(entry.size ?? 0);
|
|
287
|
+
if (declaredBytes > MAX_EXTRACTED_BYTES) {
|
|
288
|
+
refused.push(`declared size exceeds the ${MAX_EXTRACTED_BYTES}-byte ceiling`);
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
return true;
|
|
292
|
+
};
|
|
293
|
+
try {
|
|
294
|
+
await tarExtract({
|
|
295
|
+
file: archivePath,
|
|
296
|
+
cwd,
|
|
297
|
+
// tar's own guards stay on as a second layer: strict turns its warnings
|
|
298
|
+
// (e.g. its independent ".." detection) into errors; preservePaths=false
|
|
299
|
+
// keeps its absolute-path stripping; no ownership or mode replay.
|
|
300
|
+
strict: true,
|
|
301
|
+
preservePaths: false,
|
|
302
|
+
preserveOwner: false,
|
|
303
|
+
noChmod: true,
|
|
304
|
+
noMtime: true,
|
|
305
|
+
filter,
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
catch (err) {
|
|
309
|
+
if (refused.length > 0) {
|
|
310
|
+
throw new ModelCacheError("refused", `refused model archive: ${refused.join("; ")}`, err);
|
|
311
|
+
}
|
|
312
|
+
throw new ModelCacheError("incomplete", `extracting the model archive failed: ${messageOf(err)}`, err);
|
|
313
|
+
}
|
|
314
|
+
if (refused.length > 0) {
|
|
315
|
+
throw new ModelCacheError("refused", `refused model archive: ${refused.join("; ")}`);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
// ── verification + publish ────────────────────────────────────────────────────
|
|
319
|
+
/**
|
|
320
|
+
* Every required file must be a regular, non-empty file; nothing in the dir
|
|
321
|
+
* may be a symlink (belt and braces over the entry filter). Returns the size
|
|
322
|
+
* manifest for the marker.
|
|
323
|
+
*/
|
|
324
|
+
async function verifyModelDir(dir) {
|
|
325
|
+
const st = await fs.lstat(dir).catch(() => null);
|
|
326
|
+
if (!st || !st.isDirectory()) {
|
|
327
|
+
throw new ModelCacheError("incomplete", `model archive did not contain a ${MODEL_NAME}/ directory`);
|
|
328
|
+
}
|
|
329
|
+
const files = {};
|
|
330
|
+
for (const name of await fs.readdir(dir)) {
|
|
331
|
+
const s = await fs.lstat(path.join(dir, name));
|
|
332
|
+
if (s.isSymbolicLink()) {
|
|
333
|
+
throw new ModelCacheError("refused", `symlink found after extraction: ${name}`);
|
|
334
|
+
}
|
|
335
|
+
if (s.isFile())
|
|
336
|
+
files[name] = s.size;
|
|
337
|
+
}
|
|
338
|
+
const missing = REQUIRED_FILES.filter((n) => !(files[n] > 0));
|
|
339
|
+
if (missing.length > 0) {
|
|
340
|
+
throw new ModelCacheError("incomplete", `model archive is missing or has empty required file(s): ${missing.join(", ")}`);
|
|
341
|
+
}
|
|
342
|
+
return files;
|
|
343
|
+
}
|
|
344
|
+
async function writeMarker(dir, files) {
|
|
345
|
+
const marker = {
|
|
346
|
+
version: 1,
|
|
347
|
+
model: MODEL_NAME,
|
|
348
|
+
files,
|
|
349
|
+
extractedAt: new Date().toISOString(),
|
|
350
|
+
};
|
|
351
|
+
await fs.writeFile(path.join(dir, MARKER_FILE), JSON.stringify(marker, null, 2));
|
|
352
|
+
}
|
|
353
|
+
/** Rename the verified staging dir into place; tolerate a concurrent winner. */
|
|
354
|
+
async function publish(from, to) {
|
|
355
|
+
try {
|
|
356
|
+
await fs.rename(from, to);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
catch (err) {
|
|
360
|
+
// Another process may have published a complete model meanwhile — accept
|
|
361
|
+
// it. Otherwise whatever is there failed verification: replace it.
|
|
362
|
+
if (await isCompleteModelDir(to))
|
|
363
|
+
return;
|
|
364
|
+
await fs.rm(to, { recursive: true, force: true });
|
|
365
|
+
try {
|
|
366
|
+
await fs.rename(from, to);
|
|
367
|
+
}
|
|
368
|
+
catch (err2) {
|
|
369
|
+
throw new ModelCacheError("incomplete", `could not move the verified model into ${to}: ${messageOf(err2)}`, err2 ?? err);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
async function sweepStaleStaging(cacheDir) {
|
|
374
|
+
const prefix = `.${MODEL_NAME}.tmp-`;
|
|
375
|
+
const now = Date.now();
|
|
376
|
+
let names;
|
|
377
|
+
try {
|
|
378
|
+
names = await fs.readdir(cacheDir);
|
|
379
|
+
}
|
|
380
|
+
catch {
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
for (const name of names) {
|
|
384
|
+
if (!name.startsWith(prefix))
|
|
385
|
+
continue;
|
|
386
|
+
const p = path.join(cacheDir, name);
|
|
387
|
+
const st = await fs.lstat(p).catch(() => null);
|
|
388
|
+
if (st && now - st.mtimeMs > STALE_STAGING_MS) {
|
|
389
|
+
await fs.rm(p, { recursive: true, force: true }).catch(() => { });
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
function proxyConfigured() {
|
|
394
|
+
const env = process.env;
|
|
395
|
+
return Boolean(env.HTTPS_PROXY || env.https_proxy || env.HTTP_PROXY || env.http_proxy);
|
|
396
|
+
}
|
|
397
|
+
function messageOf(err) {
|
|
398
|
+
return err instanceof Error ? err.message : String(err);
|
|
399
|
+
}
|
package/dist/session/index.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
export { PROJECTS_DIR_NAME, RESERVED_SUBDIRS, SESSION_FILE_EXT, projectDir, projectKey, projectsDir, reservedDir, sessionFile, } from "./paths.js";
|
|
16
16
|
export { SessionLog } from "./log.js";
|
|
17
|
+
export { claimSession, describeHolder, ownerFile, readOwner, releaseSession, removeOwnerFile, sessionHeldBy, } from "./owner.js";
|
|
17
18
|
export { defaultExportName, exportMarkdown, } from "./export.js";
|
|
18
19
|
export { foldEvents, readEvents, readMeta, replaySession } from "./replay.js";
|
|
19
20
|
export { redactMessages } from "./redact.js";
|