@crewhaus/tool-state 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +268 -0
- package/dist/index.d.ts +43 -0
- package/dist/index.js +1858 -0
- package/dist/lib/bm25.d.ts +117 -0
- package/dist/lib/bm25.js +245 -0
- package/dist/lib/jsonl.d.ts +47 -0
- package/dist/lib/jsonl.js +102 -0
- package/dist/lib/names.d.ts +41 -0
- package/dist/lib/names.js +144 -0
- package/dist/lib/records.d.ts +66 -0
- package/dist/lib/records.js +106 -0
- package/dist/paths.d.ts +52 -0
- package/dist/paths.js +203 -0
- package/dist/store.d.ts +128 -0
- package/dist/store.js +502 -0
- package/dist/vector.d.ts +248 -0
- package/dist/vector.js +445 -0
- package/package.json +45 -0
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { type Stats } from "node:fs";
|
|
2
|
+
import { type SafePath } from "./paths";
|
|
3
|
+
/** Where state lives when the caller does not say otherwise. */
|
|
4
|
+
export declare const DEFAULT_STATE_DIR = ".crewhaus/state";
|
|
5
|
+
/** Largest single record, index or export file this package will read. */
|
|
6
|
+
export declare const MAX_FILE_BYTES: number;
|
|
7
|
+
/** How long a lock is waited for before the write is refused. */
|
|
8
|
+
export declare const LOCK_WAIT_MS = 2000;
|
|
9
|
+
/** A lock file older than this is assumed to belong to a dead process. */
|
|
10
|
+
export declare const LOCK_STALE_MS = 30000;
|
|
11
|
+
/** How deep a walk of the state directory goes before it calls it truncated. */
|
|
12
|
+
export declare const MAX_WALK_DEPTH = 32;
|
|
13
|
+
/** The outcome of reading a file that may legitimately be absent or broken. */
|
|
14
|
+
export type ReadOutcome<T> = {
|
|
15
|
+
readonly kind: "ok";
|
|
16
|
+
readonly value: T;
|
|
17
|
+
readonly stats: Stats;
|
|
18
|
+
} | {
|
|
19
|
+
readonly kind: "missing";
|
|
20
|
+
} | {
|
|
21
|
+
readonly kind: "corrupt";
|
|
22
|
+
readonly reason: string;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Resolve the state directory for this call. Throws `ToolPermissionError`
|
|
26
|
+
* when the caller points it outside the workspace.
|
|
27
|
+
*/
|
|
28
|
+
export declare function stateRoot(toolName: string, stateDir: string | undefined): SafePath;
|
|
29
|
+
/**
|
|
30
|
+
* Join segments beneath the state root and check containment BOTH ways.
|
|
31
|
+
*
|
|
32
|
+
* Lexically first — most segments come from `encodeSegment`, which cannot
|
|
33
|
+
* emit a separator, but `StateImport`'s come straight out of a document and
|
|
34
|
+
* may be anything. Then through `resolveWithin`, because a lexical check is
|
|
35
|
+
* blind to the case that actually gets exploited: an ordinary-looking
|
|
36
|
+
* `kv/cache` that is a symlink to `/tmp`, which makes a textually-contained
|
|
37
|
+
* path write outside the workspace (CWE-59). A dangling symlink is refused
|
|
38
|
+
* for the same reason — `open(…, "a")` would follow it and create the target.
|
|
39
|
+
*
|
|
40
|
+
* The returned path is the lexical one, which is what the result messages and
|
|
41
|
+
* `relativeToRoot` are expressed in; the residual check-to-use window is the
|
|
42
|
+
* usual one, and is narrower than the lifetime of any single tool call.
|
|
43
|
+
*/
|
|
44
|
+
export declare function statePath(toolName: string, root: SafePath, ...segments: string[]): string;
|
|
45
|
+
/** Path relative to the state root, `/`-separated, for echoing in results. */
|
|
46
|
+
export declare function relativeToRoot(root: SafePath, abs: string): string;
|
|
47
|
+
export declare function ensureDir(abs: string): void;
|
|
48
|
+
/** Read a UTF-8 file, reporting absence and over-size instead of throwing. */
|
|
49
|
+
export declare function readTextFile(abs: string, maxBytes?: number): ReadOutcome<string>;
|
|
50
|
+
/** Read and parse a JSON file. A parse failure is corruption, not a crash. */
|
|
51
|
+
export declare function readJsonFile(abs: string, maxBytes?: number): ReadOutcome<unknown>;
|
|
52
|
+
/**
|
|
53
|
+
* Write JSON atomically: a temporary sibling, then a rename over the target.
|
|
54
|
+
* The temporary name is derived from the pid and a counter rather than a
|
|
55
|
+
* random suffix, so a crashed run leaves a name you can recognise.
|
|
56
|
+
*/
|
|
57
|
+
export declare function writeJsonAtomic(abs: string, value: unknown): number;
|
|
58
|
+
/**
|
|
59
|
+
* Append one line with `O_APPEND`, so the kernel places it at the end of the
|
|
60
|
+
* file as a single write and two concurrent appenders cannot split each
|
|
61
|
+
* other's lines.
|
|
62
|
+
*/
|
|
63
|
+
export declare function appendLine(abs: string, line: string): void;
|
|
64
|
+
/**
|
|
65
|
+
* Create a file only if it does not exist. The exclusive create is the
|
|
66
|
+
* atomicity primitive behind `DedupeMark` and checkpoint version allocation:
|
|
67
|
+
* exactly one caller can win, with no lock involved.
|
|
68
|
+
*/
|
|
69
|
+
export declare function createExclusive(abs: string, body: string): boolean;
|
|
70
|
+
export type LockOutcome<T> = {
|
|
71
|
+
readonly ok: true;
|
|
72
|
+
readonly value: T;
|
|
73
|
+
} | {
|
|
74
|
+
readonly ok: false;
|
|
75
|
+
readonly reason: string;
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Run `body` while holding a lock file, waiting up to `waitMs` for it.
|
|
79
|
+
*
|
|
80
|
+
* Cooperative and machine-local: it serialises the read-modify-write sequences
|
|
81
|
+
* in this package against each other, which is what stops two agents in one
|
|
82
|
+
* workspace from clobbering a counter. It is not a distributed lock.
|
|
83
|
+
*/
|
|
84
|
+
export declare function withLock<T>(lockPath: string, body: () => T | Promise<T>, waitMs?: number): Promise<LockOutcome<T>>;
|
|
85
|
+
/** Names of the entries in a directory, sorted, or `[]` when it is absent. */
|
|
86
|
+
export declare function listNames(dirAbs: string, kind: "file" | "dir"): string[];
|
|
87
|
+
export type WalkedFile = {
|
|
88
|
+
/** Path relative to the walk root, `/`-separated. */
|
|
89
|
+
readonly path: string;
|
|
90
|
+
readonly abs: string;
|
|
91
|
+
readonly bytes: number;
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* Every regular file under `rootAbs`, sorted by relative path. Symlinks are
|
|
95
|
+
* skipped rather than followed: an export should carry the state directory's
|
|
96
|
+
* own bytes, never whatever a link inside it happens to point at.
|
|
97
|
+
*/
|
|
98
|
+
export declare function walkFiles(rootAbs: string, maxFiles: number, maxDepth?: number): {
|
|
99
|
+
files: WalkedFile[];
|
|
100
|
+
truncated: boolean;
|
|
101
|
+
};
|
|
102
|
+
/** The size of a file, or `undefined` when it has gone or cannot be stat'd. */
|
|
103
|
+
export declare function fileSize(abs: string): number | undefined;
|
|
104
|
+
/** Remove a file, reporting whether it was there. */
|
|
105
|
+
export declare function removeFile(abs: string): boolean;
|
|
106
|
+
/** Remove a directory tree. Used only by `StateImport` in replace mode. */
|
|
107
|
+
export declare function removeTree(abs: string): void;
|
|
108
|
+
export declare function sha256(text: string | Buffer): string;
|
|
109
|
+
/** The `errno` code of a Node filesystem error, when it has one. */
|
|
110
|
+
export declare function errorCode(err: unknown): string | undefined;
|
|
111
|
+
/**
|
|
112
|
+
* A readable sentence for an I/O failure a caller can act on (permissions, a
|
|
113
|
+
* full disk, a path that is not a directory). Anything else returns
|
|
114
|
+
* `undefined` and is re-thrown, because a bug should not be disguised as a
|
|
115
|
+
* tool result.
|
|
116
|
+
*/
|
|
117
|
+
export declare function describeIoError(err: unknown): string | undefined;
|
|
118
|
+
/**
|
|
119
|
+
* Atomically write raw bytes — the binary counterpart of `writeJsonAtomic`,
|
|
120
|
+
* used only by `StateImport` when restoring a base64-carried entry.
|
|
121
|
+
*/
|
|
122
|
+
export declare function writeBuffer(abs: string, body: Buffer): number;
|
|
123
|
+
/**
|
|
124
|
+
* Read raw bytes, for the one case text cannot carry: a binary state file.
|
|
125
|
+
* `undefined` when the file is larger than `maxBytes`, or is not a regular
|
|
126
|
+
* file, rather than pulling something unbounded into memory.
|
|
127
|
+
*/
|
|
128
|
+
export declare function readBytes(abs: string, maxBytes?: number): Buffer | undefined;
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,502 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The file-backed store beneath every tool in this package.
|
|
3
|
+
*
|
|
4
|
+
* There is no database and no server: the whole of a harness's state is a
|
|
5
|
+
* directory of small JSON and JSONL files under the workspace, which means it
|
|
6
|
+
* can be inspected with `cat`, diffed, committed, copied to another machine,
|
|
7
|
+
* and deleted by removing a folder. What this layer owns is the four things
|
|
8
|
+
* that are easy to get wrong when several agents share that directory:
|
|
9
|
+
*
|
|
10
|
+
* CONTAINMENT the state directory is resolved through `resolveSafe`, so a
|
|
11
|
+
* caller-supplied `stateDir` cannot reach outside the
|
|
12
|
+
* workspace, and every interior path is re-checked through
|
|
13
|
+
* `resolveWithin` — lexically AND against its real location,
|
|
14
|
+
* so a symlink sitting inside the state directory cannot be
|
|
15
|
+
* written through.
|
|
16
|
+
* ATOMICITY a whole-file write goes to a temporary file and is renamed
|
|
17
|
+
* over the target, so a reader never sees a half-written
|
|
18
|
+
* record and a crash mid-write leaves the old one intact.
|
|
19
|
+
* EXCLUSION read-modify-write sequences (compare-and-set, counters,
|
|
20
|
+
* sequence allocation) run under a lock file with a deadline;
|
|
21
|
+
* a lock older than `LOCK_STALE_MS` is treated as abandoned.
|
|
22
|
+
* TOLERANCE every read reports "missing", "corrupt" or "too large"
|
|
23
|
+
* rather than throwing, because a truncated file is a normal
|
|
24
|
+
* thing to find after a killed process and the caller needs to
|
|
25
|
+
* be told which file it was.
|
|
26
|
+
*
|
|
27
|
+
* The honest limits: the lock is cooperative and machine-local, so it does
|
|
28
|
+
* nothing against a process that ignores it or against two machines sharing a
|
|
29
|
+
* directory over NFS; and append ordering between processes is whatever
|
|
30
|
+
* `O_APPEND` gives, which is atomic per write on local filesystems but not
|
|
31
|
+
* guaranteed on network ones.
|
|
32
|
+
*/
|
|
33
|
+
import { createHash } from "node:crypto";
|
|
34
|
+
import { closeSync, mkdirSync, openSync, readSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeSync, } from "node:fs";
|
|
35
|
+
import * as path from "node:path";
|
|
36
|
+
import { compareStrings } from "./lib/names";
|
|
37
|
+
import { ToolPermissionError, isInside, resolveSafe, resolveWithin, toPosix, } from "./paths";
|
|
38
|
+
/** Where state lives when the caller does not say otherwise. */
|
|
39
|
+
export const DEFAULT_STATE_DIR = ".crewhaus/state";
|
|
40
|
+
/** Largest single record, index or export file this package will read. */
|
|
41
|
+
export const MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
42
|
+
/** How long a lock is waited for before the write is refused. */
|
|
43
|
+
export const LOCK_WAIT_MS = 2_000;
|
|
44
|
+
/** A lock file older than this is assumed to belong to a dead process. */
|
|
45
|
+
export const LOCK_STALE_MS = 30_000;
|
|
46
|
+
/** How deep a walk of the state directory goes before it calls it truncated. */
|
|
47
|
+
export const MAX_WALK_DEPTH = 32;
|
|
48
|
+
/**
|
|
49
|
+
* Resolve the state directory for this call. Throws `ToolPermissionError`
|
|
50
|
+
* when the caller points it outside the workspace.
|
|
51
|
+
*/
|
|
52
|
+
export function stateRoot(toolName, stateDir) {
|
|
53
|
+
return resolveSafe(toolName, stateDir ?? DEFAULT_STATE_DIR);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Join segments beneath the state root and check containment BOTH ways.
|
|
57
|
+
*
|
|
58
|
+
* Lexically first — most segments come from `encodeSegment`, which cannot
|
|
59
|
+
* emit a separator, but `StateImport`'s come straight out of a document and
|
|
60
|
+
* may be anything. Then through `resolveWithin`, because a lexical check is
|
|
61
|
+
* blind to the case that actually gets exploited: an ordinary-looking
|
|
62
|
+
* `kv/cache` that is a symlink to `/tmp`, which makes a textually-contained
|
|
63
|
+
* path write outside the workspace (CWE-59). A dangling symlink is refused
|
|
64
|
+
* for the same reason — `open(…, "a")` would follow it and create the target.
|
|
65
|
+
*
|
|
66
|
+
* The returned path is the lexical one, which is what the result messages and
|
|
67
|
+
* `relativeToRoot` are expressed in; the residual check-to-use window is the
|
|
68
|
+
* usual one, and is narrower than the lifetime of any single tool call.
|
|
69
|
+
*/
|
|
70
|
+
export function statePath(toolName, root, ...segments) {
|
|
71
|
+
const label = segments.join("/");
|
|
72
|
+
const joined = path.join(root.abs, ...segments);
|
|
73
|
+
if (!isInside(root.abs, joined))
|
|
74
|
+
throw new ToolPermissionError(toolName, label);
|
|
75
|
+
resolveWithin(toolName, root.real, joined, label);
|
|
76
|
+
return joined;
|
|
77
|
+
}
|
|
78
|
+
/** Path relative to the state root, `/`-separated, for echoing in results. */
|
|
79
|
+
export function relativeToRoot(root, abs) {
|
|
80
|
+
return toPosix(path.relative(root.abs, abs));
|
|
81
|
+
}
|
|
82
|
+
export function ensureDir(abs) {
|
|
83
|
+
mkdirSync(abs, { recursive: true });
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Read a file into a buffer sized from its own stat, never larger than
|
|
87
|
+
* `maxBytes + 1`.
|
|
88
|
+
*
|
|
89
|
+
* `readFileSync` sizes itself from the file, so a file that grows between the
|
|
90
|
+
* stat that approved it and the read that follows is read in full — the cap
|
|
91
|
+
* is then a cap on what was CHECKED, not on what is allocated. One byte over
|
|
92
|
+
* the expected size is enough to detect the growth and refuse it.
|
|
93
|
+
*/
|
|
94
|
+
function readCapped(abs, expectedBytes, maxBytes) {
|
|
95
|
+
const room = Math.min(expectedBytes, maxBytes) + 1;
|
|
96
|
+
const fd = openSync(abs, "r");
|
|
97
|
+
try {
|
|
98
|
+
const buffer = Buffer.allocUnsafe(room);
|
|
99
|
+
let filled = 0;
|
|
100
|
+
// `read(2)` is allowed to return fewer bytes than asked for, so a single
|
|
101
|
+
// call could silently truncate a large file into "valid" content. Loop
|
|
102
|
+
// until it is full or the file ends.
|
|
103
|
+
for (;;) {
|
|
104
|
+
const read = readSync(fd, buffer, filled, room - filled, filled);
|
|
105
|
+
if (read <= 0)
|
|
106
|
+
break;
|
|
107
|
+
filled += read;
|
|
108
|
+
if (filled >= room)
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
return buffer.subarray(0, filled);
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
closeSync(fd);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** Read a UTF-8 file, reporting absence and over-size instead of throwing. */
|
|
118
|
+
export function readTextFile(abs, maxBytes = MAX_FILE_BYTES) {
|
|
119
|
+
let stats;
|
|
120
|
+
try {
|
|
121
|
+
stats = statSync(abs);
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
if (errorCode(err) === "ENOENT")
|
|
125
|
+
return { kind: "missing" };
|
|
126
|
+
throw err;
|
|
127
|
+
}
|
|
128
|
+
if (stats.isDirectory())
|
|
129
|
+
return { kind: "corrupt", reason: "expected a file but found a directory" };
|
|
130
|
+
// A FIFO stats as size 0, and `readFileSync` on one BLOCKS UNTIL A WRITER
|
|
131
|
+
// OPENS IT — forever, with no deadline anywhere in this package to stop it.
|
|
132
|
+
// Sockets and devices are no better. Only a regular file is read.
|
|
133
|
+
if (!stats.isFile())
|
|
134
|
+
return { kind: "corrupt", reason: "not a regular file, so it is not something to read" };
|
|
135
|
+
if (stats.size > maxBytes) {
|
|
136
|
+
return {
|
|
137
|
+
kind: "corrupt",
|
|
138
|
+
reason: `file is ${stats.size} bytes, over the ${maxBytes} limit for this tool`,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
// Read at most `maxBytes + 1` so a file that grew between the stat and the
|
|
142
|
+
// read cannot pull more than the cap into memory; the extra byte is what
|
|
143
|
+
// detects the growth.
|
|
144
|
+
let buffer;
|
|
145
|
+
try {
|
|
146
|
+
buffer = readCapped(abs, stats.size, maxBytes);
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
// Deleted between the stat and the open: absent is the honest answer, and
|
|
150
|
+
// a listing that met the file a moment ago should not fail over it.
|
|
151
|
+
if (errorCode(err) === "ENOENT")
|
|
152
|
+
return { kind: "missing" };
|
|
153
|
+
throw err;
|
|
154
|
+
}
|
|
155
|
+
if (buffer.byteLength > stats.size || buffer.byteLength > maxBytes) {
|
|
156
|
+
return {
|
|
157
|
+
kind: "corrupt",
|
|
158
|
+
reason: "file changed size while it was being read — retry once it has settled",
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
if (buffer.includes(0)) {
|
|
162
|
+
return {
|
|
163
|
+
kind: "corrupt",
|
|
164
|
+
reason: "file contains NUL bytes, so it is not the text this store writes",
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
return { kind: "ok", value: buffer.toString("utf8"), stats };
|
|
168
|
+
}
|
|
169
|
+
/** Read and parse a JSON file. A parse failure is corruption, not a crash. */
|
|
170
|
+
export function readJsonFile(abs, maxBytes = MAX_FILE_BYTES) {
|
|
171
|
+
const text = readTextFile(abs, maxBytes);
|
|
172
|
+
if (text.kind !== "ok")
|
|
173
|
+
return text;
|
|
174
|
+
try {
|
|
175
|
+
return { kind: "ok", value: JSON.parse(text.value), stats: text.stats };
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
return { kind: "corrupt", reason: `not valid JSON: ${err.message}` };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
let tempCounter = 0;
|
|
182
|
+
/**
|
|
183
|
+
* Write JSON atomically: a temporary sibling, then a rename over the target.
|
|
184
|
+
* The temporary name is derived from the pid and a counter rather than a
|
|
185
|
+
* random suffix, so a crashed run leaves a name you can recognise.
|
|
186
|
+
*/
|
|
187
|
+
export function writeJsonAtomic(abs, value) {
|
|
188
|
+
ensureDir(path.dirname(abs));
|
|
189
|
+
const body = `${JSON.stringify(value)}\n`;
|
|
190
|
+
tempCounter += 1;
|
|
191
|
+
const temp = `${abs}.tmp-${process.pid}-${tempCounter}`;
|
|
192
|
+
const fd = openSync(temp, "wx");
|
|
193
|
+
try {
|
|
194
|
+
writeSync(fd, body);
|
|
195
|
+
}
|
|
196
|
+
finally {
|
|
197
|
+
closeSync(fd);
|
|
198
|
+
}
|
|
199
|
+
try {
|
|
200
|
+
renameSync(temp, abs);
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
try {
|
|
204
|
+
unlinkSync(temp);
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
// the rename failed and so did the cleanup; the original is still intact
|
|
208
|
+
}
|
|
209
|
+
throw err;
|
|
210
|
+
}
|
|
211
|
+
return Buffer.byteLength(body, "utf8");
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Append one line with `O_APPEND`, so the kernel places it at the end of the
|
|
215
|
+
* file as a single write and two concurrent appenders cannot split each
|
|
216
|
+
* other's lines.
|
|
217
|
+
*/
|
|
218
|
+
export function appendLine(abs, line) {
|
|
219
|
+
ensureDir(path.dirname(abs));
|
|
220
|
+
const fd = openSync(abs, "a");
|
|
221
|
+
try {
|
|
222
|
+
writeSync(fd, line);
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
closeSync(fd);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Create a file only if it does not exist. The exclusive create is the
|
|
230
|
+
* atomicity primitive behind `DedupeMark` and checkpoint version allocation:
|
|
231
|
+
* exactly one caller can win, with no lock involved.
|
|
232
|
+
*/
|
|
233
|
+
export function createExclusive(abs, body) {
|
|
234
|
+
ensureDir(path.dirname(abs));
|
|
235
|
+
let fd;
|
|
236
|
+
try {
|
|
237
|
+
fd = openSync(abs, "wx");
|
|
238
|
+
}
|
|
239
|
+
catch (err) {
|
|
240
|
+
if (errorCode(err) === "EEXIST")
|
|
241
|
+
return false;
|
|
242
|
+
throw err;
|
|
243
|
+
}
|
|
244
|
+
try {
|
|
245
|
+
writeSync(fd, body);
|
|
246
|
+
}
|
|
247
|
+
finally {
|
|
248
|
+
closeSync(fd);
|
|
249
|
+
}
|
|
250
|
+
return true;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Run `body` while holding a lock file, waiting up to `waitMs` for it.
|
|
254
|
+
*
|
|
255
|
+
* Cooperative and machine-local: it serialises the read-modify-write sequences
|
|
256
|
+
* in this package against each other, which is what stops two agents in one
|
|
257
|
+
* workspace from clobbering a counter. It is not a distributed lock.
|
|
258
|
+
*/
|
|
259
|
+
export async function withLock(lockPath, body, waitMs = LOCK_WAIT_MS) {
|
|
260
|
+
ensureDir(path.dirname(lockPath));
|
|
261
|
+
const deadline = Date.now() + waitMs;
|
|
262
|
+
for (;;) {
|
|
263
|
+
try {
|
|
264
|
+
const fd = openSync(lockPath, "wx");
|
|
265
|
+
writeSync(fd, `${process.pid}\n`);
|
|
266
|
+
closeSync(fd);
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
catch (err) {
|
|
270
|
+
if (errorCode(err) !== "EEXIST")
|
|
271
|
+
throw err;
|
|
272
|
+
const age = lockAgeMs(lockPath);
|
|
273
|
+
if (age !== undefined && age > LOCK_STALE_MS) {
|
|
274
|
+
try {
|
|
275
|
+
unlinkSync(lockPath);
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
// somebody else broke it first; loop round and try to take it
|
|
279
|
+
}
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
if (Date.now() >= deadline) {
|
|
283
|
+
return {
|
|
284
|
+
ok: false,
|
|
285
|
+
reason: `another writer has held the lock on ${path.basename(lockPath)} for more than ${waitMs}ms — retry, or remove the lock file if you are sure no other agent is running`,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
await Bun.sleep(5);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
return { ok: true, value: await body() };
|
|
293
|
+
}
|
|
294
|
+
finally {
|
|
295
|
+
try {
|
|
296
|
+
unlinkSync(lockPath);
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
// already gone (broken as stale by another waiter); nothing to undo
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
function lockAgeMs(lockPath) {
|
|
304
|
+
try {
|
|
305
|
+
return Date.now() - statSync(lockPath).mtimeMs;
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
return undefined;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
/** Names of the entries in a directory, sorted, or `[]` when it is absent. */
|
|
312
|
+
export function listNames(dirAbs, kind) {
|
|
313
|
+
let entries;
|
|
314
|
+
try {
|
|
315
|
+
entries = readdirSync(dirAbs, { withFileTypes: true });
|
|
316
|
+
}
|
|
317
|
+
catch (err) {
|
|
318
|
+
if (errorCode(err) === "ENOENT" || errorCode(err) === "ENOTDIR")
|
|
319
|
+
return [];
|
|
320
|
+
throw err;
|
|
321
|
+
}
|
|
322
|
+
const names = [];
|
|
323
|
+
for (const entry of entries) {
|
|
324
|
+
if (entry.name.includes(".tmp-") || entry.name.endsWith(".lock"))
|
|
325
|
+
continue;
|
|
326
|
+
if (kind === "file" ? entry.isFile() : entry.isDirectory())
|
|
327
|
+
names.push(entry.name);
|
|
328
|
+
}
|
|
329
|
+
return names.sort(compareStrings);
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Every regular file under `rootAbs`, sorted by relative path. Symlinks are
|
|
333
|
+
* skipped rather than followed: an export should carry the state directory's
|
|
334
|
+
* own bytes, never whatever a link inside it happens to point at.
|
|
335
|
+
*/
|
|
336
|
+
export function walkFiles(rootAbs, maxFiles, maxDepth = MAX_WALK_DEPTH) {
|
|
337
|
+
const files = [];
|
|
338
|
+
let truncated = false;
|
|
339
|
+
const visit = (dir, prefix, depth) => {
|
|
340
|
+
if (truncated)
|
|
341
|
+
return;
|
|
342
|
+
// A state directory is three levels deep. Anything far past that is a
|
|
343
|
+
// tree somebody else made, and the walk stops rather than descending it
|
|
344
|
+
// for however long it happens to go on.
|
|
345
|
+
if (depth > maxDepth) {
|
|
346
|
+
truncated = true;
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
let entries;
|
|
350
|
+
try {
|
|
351
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
352
|
+
}
|
|
353
|
+
catch (err) {
|
|
354
|
+
if (errorCode(err) === "ENOENT")
|
|
355
|
+
return;
|
|
356
|
+
throw err;
|
|
357
|
+
}
|
|
358
|
+
const sorted = entries.slice().sort((a, b) => compareStrings(a.name, b.name));
|
|
359
|
+
for (const entry of sorted) {
|
|
360
|
+
if (entry.isSymbolicLink())
|
|
361
|
+
continue;
|
|
362
|
+
const abs = path.join(dir, entry.name);
|
|
363
|
+
const rel = prefix === "" ? entry.name : `${prefix}/${entry.name}`;
|
|
364
|
+
if (entry.isDirectory()) {
|
|
365
|
+
visit(abs, rel, depth + 1);
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (!entry.isFile())
|
|
369
|
+
continue;
|
|
370
|
+
if (entry.name.includes(".tmp-") || entry.name.endsWith(".lock"))
|
|
371
|
+
continue;
|
|
372
|
+
if (files.length >= maxFiles) {
|
|
373
|
+
truncated = true;
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
const bytes = fileSize(abs);
|
|
377
|
+
// Deleted between the readdir and the stat: it is not in the export,
|
|
378
|
+
// and that is not a reason to fail the whole export.
|
|
379
|
+
if (bytes === undefined)
|
|
380
|
+
continue;
|
|
381
|
+
files.push({ path: rel, abs, bytes });
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
visit(rootAbs, "", 0);
|
|
385
|
+
return { files, truncated };
|
|
386
|
+
}
|
|
387
|
+
/** The size of a file, or `undefined` when it has gone or cannot be stat'd. */
|
|
388
|
+
export function fileSize(abs) {
|
|
389
|
+
try {
|
|
390
|
+
const stats = statSync(abs);
|
|
391
|
+
return stats.isFile() ? stats.size : undefined;
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return undefined;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
/** Remove a file, reporting whether it was there. */
|
|
398
|
+
export function removeFile(abs) {
|
|
399
|
+
try {
|
|
400
|
+
unlinkSync(abs);
|
|
401
|
+
return true;
|
|
402
|
+
}
|
|
403
|
+
catch (err) {
|
|
404
|
+
if (errorCode(err) === "ENOENT")
|
|
405
|
+
return false;
|
|
406
|
+
throw err;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
/** Remove a directory tree. Used only by `StateImport` in replace mode. */
|
|
410
|
+
export function removeTree(abs) {
|
|
411
|
+
rmSync(abs, { recursive: true, force: true });
|
|
412
|
+
}
|
|
413
|
+
export function sha256(text) {
|
|
414
|
+
return createHash("sha256").update(text).digest("hex");
|
|
415
|
+
}
|
|
416
|
+
/** The `errno` code of a Node filesystem error, when it has one. */
|
|
417
|
+
export function errorCode(err) {
|
|
418
|
+
if (typeof err !== "object" || err === null)
|
|
419
|
+
return undefined;
|
|
420
|
+
const code = err.code;
|
|
421
|
+
return typeof code === "string" ? code : undefined;
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* A readable sentence for an I/O failure a caller can act on (permissions, a
|
|
425
|
+
* full disk, a path that is not a directory). Anything else returns
|
|
426
|
+
* `undefined` and is re-thrown, because a bug should not be disguised as a
|
|
427
|
+
* tool result.
|
|
428
|
+
*/
|
|
429
|
+
export function describeIoError(err) {
|
|
430
|
+
const code = errorCode(err);
|
|
431
|
+
if (code === undefined)
|
|
432
|
+
return undefined;
|
|
433
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
434
|
+
switch (code) {
|
|
435
|
+
case "EACCES":
|
|
436
|
+
case "EPERM":
|
|
437
|
+
return `permission denied: ${message}`;
|
|
438
|
+
case "ENOSPC":
|
|
439
|
+
return `no space left on the device: ${message}`;
|
|
440
|
+
case "EROFS":
|
|
441
|
+
return `the filesystem is read-only: ${message}`;
|
|
442
|
+
case "ENOTDIR":
|
|
443
|
+
return `a path component is not a directory: ${message}`;
|
|
444
|
+
case "EISDIR":
|
|
445
|
+
return `expected a file but found a directory: ${message}`;
|
|
446
|
+
case "EMFILE":
|
|
447
|
+
case "ENFILE":
|
|
448
|
+
return `too many open files: ${message}`;
|
|
449
|
+
case "ENAMETOOLONG":
|
|
450
|
+
return `the resulting filename is too long: ${message}`;
|
|
451
|
+
default:
|
|
452
|
+
return undefined;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Atomically write raw bytes — the binary counterpart of `writeJsonAtomic`,
|
|
457
|
+
* used only by `StateImport` when restoring a base64-carried entry.
|
|
458
|
+
*/
|
|
459
|
+
export function writeBuffer(abs, body) {
|
|
460
|
+
ensureDir(path.dirname(abs));
|
|
461
|
+
tempCounter += 1;
|
|
462
|
+
const temp = `${abs}.tmp-${process.pid}-${tempCounter}`;
|
|
463
|
+
const fd = openSync(temp, "wx");
|
|
464
|
+
try {
|
|
465
|
+
writeSync(fd, body);
|
|
466
|
+
}
|
|
467
|
+
finally {
|
|
468
|
+
closeSync(fd);
|
|
469
|
+
}
|
|
470
|
+
try {
|
|
471
|
+
renameSync(temp, abs);
|
|
472
|
+
}
|
|
473
|
+
catch (err) {
|
|
474
|
+
try {
|
|
475
|
+
unlinkSync(temp);
|
|
476
|
+
}
|
|
477
|
+
catch {
|
|
478
|
+
// the rename failed and so did the cleanup; the original is still intact
|
|
479
|
+
}
|
|
480
|
+
throw err;
|
|
481
|
+
}
|
|
482
|
+
return body.byteLength;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Read raw bytes, for the one case text cannot carry: a binary state file.
|
|
486
|
+
* `undefined` when the file is larger than `maxBytes`, or is not a regular
|
|
487
|
+
* file, rather than pulling something unbounded into memory.
|
|
488
|
+
*/
|
|
489
|
+
export function readBytes(abs, maxBytes = MAX_FILE_BYTES) {
|
|
490
|
+
try {
|
|
491
|
+
const stats = statSync(abs);
|
|
492
|
+
if (!stats.isFile() || stats.size > maxBytes)
|
|
493
|
+
return undefined;
|
|
494
|
+
const buffer = readCapped(abs, stats.size, maxBytes);
|
|
495
|
+
return buffer.byteLength > stats.size ? undefined : buffer;
|
|
496
|
+
}
|
|
497
|
+
catch (err) {
|
|
498
|
+
if (errorCode(err) === "ENOENT")
|
|
499
|
+
return undefined;
|
|
500
|
+
throw err;
|
|
501
|
+
}
|
|
502
|
+
}
|