dsh-boost 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +29 -0
- package/README.en.md +476 -0
- package/README.md +408 -2
- package/cordis.patch.yml +40 -0
- package/dist/admission/fixed-cost.d.ts +23 -0
- package/dist/admission/fixed-cost.js +61 -0
- package/dist/admission/fixed-cost.js.map +1 -0
- package/dist/admission/ledger.d.ts +113 -0
- package/dist/admission/ledger.js +176 -0
- package/dist/admission/ledger.js.map +1 -0
- package/dist/admission/retrieval.d.ts +65 -0
- package/dist/admission/retrieval.js +162 -0
- package/dist/admission/retrieval.js.map +1 -0
- package/dist/admission/usage.d.ts +75 -0
- package/dist/admission/usage.js +124 -0
- package/dist/admission/usage.js.map +1 -0
- package/dist/config.d.ts +43 -0
- package/dist/config.js +103 -0
- package/dist/config.js.map +1 -0
- package/dist/gate/binding.d.ts +113 -0
- package/dist/gate/binding.js +297 -0
- package/dist/gate/binding.js.map +1 -0
- package/dist/gate/decide.d.ts +82 -0
- package/dist/gate/decide.js +163 -0
- package/dist/gate/decide.js.map +1 -0
- package/dist/gate/guard.d.ts +69 -0
- package/dist/gate/guard.js +195 -0
- package/dist/gate/guard.js.map +1 -0
- package/dist/gate/invalidate.d.ts +112 -0
- package/dist/gate/invalidate.js +111 -0
- package/dist/gate/invalidate.js.map +1 -0
- package/dist/gate/plan.d.ts +86 -0
- package/dist/gate/plan.js +222 -0
- package/dist/gate/plan.js.map +1 -0
- package/dist/hooks/clock.d.ts +68 -0
- package/dist/hooks/clock.js +82 -0
- package/dist/hooks/clock.js.map +1 -0
- package/dist/hooks/execute.d.ts +79 -0
- package/dist/hooks/execute.js +128 -0
- package/dist/hooks/execute.js.map +1 -0
- package/dist/hooks/llm-stream.d.ts +276 -0
- package/dist/hooks/llm-stream.js +1008 -0
- package/dist/hooks/llm-stream.js.map +1 -0
- package/dist/hooks/pending.d.ts +89 -0
- package/dist/hooks/pending.js +115 -0
- package/dist/hooks/pending.js.map +1 -0
- package/dist/hooks/post-execute.d.ts +247 -0
- package/dist/hooks/post-execute.js +1776 -0
- package/dist/hooks/post-execute.js.map +1 -0
- package/dist/hooks/session-state.d.ts +158 -0
- package/dist/hooks/session-state.js +168 -0
- package/dist/hooks/session-state.js.map +1 -0
- package/dist/hooks/synthesize.d.ts +74 -0
- package/dist/hooks/synthesize.js +100 -0
- package/dist/hooks/synthesize.js.map +1 -0
- package/dist/host/loader.d.ts +23 -0
- package/dist/host/loader.js +31 -0
- package/dist/host/loader.js.map +1 -0
- package/dist/host/platform.d.ts +18 -0
- package/dist/host/platform.js +19 -0
- package/dist/host/platform.js.map +1 -0
- package/dist/host/root.d.ts +18 -0
- package/dist/host/root.js +43 -0
- package/dist/host/root.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +866 -0
- package/dist/index.js.map +1 -0
- package/dist/projection/apply.d.ts +174 -0
- package/dist/projection/apply.js +137 -0
- package/dist/projection/apply.js.map +1 -0
- package/dist/projection/canonicalise.d.ts +43 -0
- package/dist/projection/canonicalise.js +43 -0
- package/dist/projection/canonicalise.js.map +1 -0
- package/dist/projection/counters.d.ts +65 -0
- package/dist/projection/counters.js +88 -0
- package/dist/projection/counters.js.map +1 -0
- package/dist/projection/truncate.d.ts +47 -0
- package/dist/projection/truncate.js +57 -0
- package/dist/projection/truncate.js.map +1 -0
- package/dist/prompt/protocol.d.ts +45 -0
- package/dist/prompt/protocol.js +56 -0
- package/dist/prompt/protocol.js.map +1 -0
- package/dist/receipts/store.d.ts +198 -0
- package/dist/receipts/store.js +734 -0
- package/dist/receipts/store.js.map +1 -0
- package/dist/scope/doc.d.ts +117 -0
- package/dist/scope/doc.js +158 -0
- package/dist/scope/doc.js.map +1 -0
- package/dist/scope/resolve.d.ts +91 -0
- package/dist/scope/resolve.js +146 -0
- package/dist/scope/resolve.js.map +1 -0
- package/dist/scope/store.d.ts +135 -0
- package/dist/scope/store.js +573 -0
- package/dist/scope/store.js.map +1 -0
- package/dist/selector/jev.d.ts +93 -0
- package/dist/selector/jev.js +171 -0
- package/dist/selector/jev.js.map +1 -0
- package/dist/tools/boost.d.ts +367 -0
- package/dist/tools/boost.js +768 -0
- package/dist/tools/boost.js.map +1 -0
- package/dist/tools/family.d.ts +26 -0
- package/dist/tools/family.js +26 -0
- package/dist/tools/family.js.map +1 -0
- package/dist/types.d.ts +113 -0
- package/dist/types.js +21 -0
- package/dist/types.js.map +1 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +75 -4
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only receipt store.
|
|
3
|
+
*
|
|
4
|
+
* Every decision the plugin makes is one JSON line under
|
|
5
|
+
* `<root>/receipts/<encodeSegment(sessionId)>.jsonl`: the shadow week is
|
|
6
|
+
* analysed offline from these files, and the plugin is audited from them after
|
|
7
|
+
* the fact. The shard name percent-encodes every character outside
|
|
8
|
+
* `[A-Za-z0-9_-]` (reversibly, one `%XX` per UTF-8 byte) because a host-supplied
|
|
9
|
+
* session id may contain `/` or `..` and the store must never write outside
|
|
10
|
+
* `receipts/`; the receipt body keeps the original id. The store is therefore
|
|
11
|
+
* append-only (a read never writes; it ignores an unfinished tail), crash-safe
|
|
12
|
+
* (a line that was never finished is not a receipt), bounded (current + `.1`,
|
|
13
|
+
* hard limits), and fail-closed (three consecutive write failures stop
|
|
14
|
+
* substitution rather than running without an audit trail).
|
|
15
|
+
*
|
|
16
|
+
* ## Write path
|
|
17
|
+
*
|
|
18
|
+
* A receipt is serialised and appended in one `appendFile` call, rotated before
|
|
19
|
+
* the write when the current shard reached `maxBytes`/`maxLines`, and fsynced
|
|
20
|
+
* only every `fsyncEveryLines` lines (or on `flush()` — the assembly flushes at
|
|
21
|
+
* turn end, at the host's session/flush checkpoint and before session-end
|
|
22
|
+
* eviction). A write failure never rejects: the line is buffered in memory in
|
|
23
|
+
* order, `failures()` counts the *consecutive* failures, and a later successful
|
|
24
|
+
* append replays the buffer and resets the count. The buffer is bounded, and
|
|
25
|
+
* once the fail-closed `stopped()` threshold is reached it is dropped instead
|
|
26
|
+
* of growing for the duration of the outage. `disposeSession` evicts a dead
|
|
27
|
+
* session's shard state and records the id in a bounded set, so a hook that
|
|
28
|
+
* lands after `session/disposed` records nothing.
|
|
29
|
+
*
|
|
30
|
+
* ## Read path
|
|
31
|
+
*
|
|
32
|
+
* `.1` is read before the current shard, so `scan` yields oldest receipts
|
|
33
|
+
* first. A trailing line without `\n` is **ignored** and the shard is left
|
|
34
|
+
* untouched: a read is concurrent with the writer, and truncating from a
|
|
35
|
+
* snapshot can cut a line another append just completed. Only the next write
|
|
36
|
+
* path repairs the shard (`stateFor` truncates to the last newline before
|
|
37
|
+
* appending). A corrupt line in the middle is skipped. Rotated shards are read
|
|
38
|
+
* through the same path; nothing ever appends to `.1`, so its half line simply
|
|
39
|
+
* stays ignored.
|
|
40
|
+
*
|
|
41
|
+
* ## Retention
|
|
42
|
+
*
|
|
43
|
+
* Shards are an audit trail and are never pruned: one shard set per session
|
|
44
|
+
* stays on disk for the life of the root. A host that never emits
|
|
45
|
+
* `session/disposed` therefore leaves the per-session maps (`states`,
|
|
46
|
+
* `disposed`) and `scan`'s shard list growing with the session count. The maps
|
|
47
|
+
* are bounded per session and evicted by `disposeSession`; a whole-store
|
|
48
|
+
* `scan` is additionally capped at {@link MAX_SCAN_SHARDS} sorted shards so
|
|
49
|
+
* `boost status` cannot walk an unbounded directory, while `scan(sessionId)`
|
|
50
|
+
* still addresses any one shard. The cap drops shards from that aggregate
|
|
51
|
+
* scan only; nothing is deleted.
|
|
52
|
+
*
|
|
53
|
+
* The store imports only `node:` builtins and knows no host services: the
|
|
54
|
+
* caller resolves the root (the plugin passes `$DSH_HOME/boost`).
|
|
55
|
+
*
|
|
56
|
+
* @module dsh-boost/receipts/store
|
|
57
|
+
*/
|
|
58
|
+
import { createHash } from 'node:crypto';
|
|
59
|
+
import { appendFile, mkdir, open, readdir, readFile, rename, stat, truncate } from 'node:fs/promises';
|
|
60
|
+
import { dirname, join } from 'node:path';
|
|
61
|
+
const RECEIPTS_DIR = 'receipts';
|
|
62
|
+
const CURRENT_SUFFIX = '.jsonl';
|
|
63
|
+
const ROTATED_SUFFIX = '.1.jsonl';
|
|
64
|
+
/** `preview` is the first 256 code points, never the full receipt text (design §5.4). */
|
|
65
|
+
const PREVIEW_CODE_POINTS = 256;
|
|
66
|
+
const DEFAULT_FSYNC_EVERY_LINES = 100;
|
|
67
|
+
const DEFAULT_MAX_BYTES = 8 * 1024 * 1024;
|
|
68
|
+
const DEFAULT_MAX_LINES = 20_000;
|
|
69
|
+
/** Fail-closed threshold: after this many *consecutive* write failures, stop substituting. */
|
|
70
|
+
const STOPPED_AFTER_FAILURES = 3;
|
|
71
|
+
/** `args` is model-written; cap what a receipt line carries (identity hashes the full value). */
|
|
72
|
+
const MAX_RECEIPT_ARGS_CODE_POINTS = 2048;
|
|
73
|
+
/** Hard cap on the in-memory replay buffer (see `recordFailure`). */
|
|
74
|
+
const MAX_PENDING_LINES = 100;
|
|
75
|
+
/**
|
|
76
|
+
* Depth bound for model-written `args`.
|
|
77
|
+
*
|
|
78
|
+
* The plan parser refuses args deeper than 64 levels, but a call that did not
|
|
79
|
+
* come from a plan (the host's own `exec.arguments`) can be arbitrarily deep.
|
|
80
|
+
* Walking it recursively would throw a `RangeError` and lose the receipt, and
|
|
81
|
+
* `JSON.stringify` would throw the same way; deeper subtrees are replaced by
|
|
82
|
+
* `{ truncated_depth: 64 }` before hashing or storing. The cut is visible in
|
|
83
|
+
* the receipt as `args_depth_truncated: true`.
|
|
84
|
+
*/
|
|
85
|
+
const MAX_RECEIPT_ARGS_DEPTH = 64;
|
|
86
|
+
/**
|
|
87
|
+
* Shards a whole-store `scan` visits at most.
|
|
88
|
+
*
|
|
89
|
+
* Shards are never deleted (the audit trail is the product), so a long-lived
|
|
90
|
+
* host accumulates one per session forever. `scan()` with no id is the
|
|
91
|
+
* status/report path; capping it keeps that path's work and memory bounded.
|
|
92
|
+
* The kept shards are the first N sorted encoded names; named scans are not
|
|
93
|
+
* capped.
|
|
94
|
+
*/
|
|
95
|
+
const MAX_SCAN_SHARDS = 256;
|
|
96
|
+
/**
|
|
97
|
+
* How many disposed session ids the store remembers.
|
|
98
|
+
*
|
|
99
|
+
* A late `post-execute` can land seconds after `session/disposed` (a selector
|
|
100
|
+
* call may take 20 s) and must not write a receipt into a closed session. The
|
|
101
|
+
* set is FIFO-bounded so a long-lived process cannot grow it without limit; a
|
|
102
|
+
* session id is unique, so evicting the oldest is safe.
|
|
103
|
+
*/
|
|
104
|
+
const MAX_DISPOSED_SESSIONS = 256;
|
|
105
|
+
const NEWLINE = 0x0a;
|
|
106
|
+
/**
|
|
107
|
+
* `sha256(stable JSON)` of the fields that identify a call, first 16 hex chars.
|
|
108
|
+
*
|
|
109
|
+
* `args` is part of the payload on purpose: without it, two different calls of
|
|
110
|
+
* the same tool in one turn would collide. A missing `args` hashes like `{}`,
|
|
111
|
+
* and object keys are sorted recursively so insertion order cannot change the
|
|
112
|
+
* hash. An own `__proto__` key is an ordinary key, not the prototype setter.
|
|
113
|
+
*
|
|
114
|
+
* Args deeper than {@link MAX_RECEIPT_ARGS_DEPTH} are cut with a marker before
|
|
115
|
+
* hashing, exactly as they are before storing: otherwise the recursive walk or
|
|
116
|
+
* `JSON.stringify` would throw and the receipt (the audit evidence) would be
|
|
117
|
+
* lost. Two trees that differ only below the bound therefore hash alike; the
|
|
118
|
+
* plan parser refuses such trees, and a call whose args are deeper did not come
|
|
119
|
+
* from a plan.
|
|
120
|
+
*/
|
|
121
|
+
export function receiptHash(r) {
|
|
122
|
+
const payload = {
|
|
123
|
+
sessionId: r.sessionId,
|
|
124
|
+
turn: r.turn,
|
|
125
|
+
stepId: r.stepId,
|
|
126
|
+
tool: r.tool,
|
|
127
|
+
args: boundArgs(r.args ?? {}).value,
|
|
128
|
+
};
|
|
129
|
+
return createHash('sha256').update(JSON.stringify(payload)).digest('hex').slice(0, 16);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* The sorted, depth-bounded copy of `value`.
|
|
133
|
+
*
|
|
134
|
+
* Keys are sorted recursively so insertion order cannot change the hash;
|
|
135
|
+
* `undefined` members are dropped as `JSON.stringify` would. The copy is built
|
|
136
|
+
* with `defineProperty`-equivalent semantics (`Object.create(null)`) because
|
|
137
|
+
* plain assignment would let an own `__proto__` key hit the prototype setter and
|
|
138
|
+
* vanish, hashing two different calls alike.
|
|
139
|
+
*
|
|
140
|
+
* Recursion is safe because the depth is capped at
|
|
141
|
+
* {@link MAX_RECEIPT_ARGS_DEPTH}: at most 65 frames regardless of the input.
|
|
142
|
+
* A deeper subtree becomes `{ truncated_depth: 64 }`.
|
|
143
|
+
*/
|
|
144
|
+
function sortKeys(value, state, depth) {
|
|
145
|
+
if (depth > MAX_RECEIPT_ARGS_DEPTH) {
|
|
146
|
+
state.depthTruncated = true;
|
|
147
|
+
return { truncated_depth: MAX_RECEIPT_ARGS_DEPTH };
|
|
148
|
+
}
|
|
149
|
+
if (Array.isArray(value))
|
|
150
|
+
return value.map((child) => sortKeys(child, state, depth + 1));
|
|
151
|
+
if (value !== null && typeof value === 'object') {
|
|
152
|
+
const sorted = Object.create(null);
|
|
153
|
+
for (const key of Object.keys(value).sort()) {
|
|
154
|
+
const child = value[key];
|
|
155
|
+
if (child !== undefined)
|
|
156
|
+
sorted[key] = sortKeys(child, state, depth + 1);
|
|
157
|
+
}
|
|
158
|
+
return sorted;
|
|
159
|
+
}
|
|
160
|
+
return value;
|
|
161
|
+
}
|
|
162
|
+
/** Bound and sort `value`, reporting whether the depth cap cut anything. */
|
|
163
|
+
function boundArgs(value) {
|
|
164
|
+
const state = { depthTruncated: false };
|
|
165
|
+
return { value: sortKeys(value, state, 1), depthTruncated: state.depthTruncated };
|
|
166
|
+
}
|
|
167
|
+
/** First 256 code points; never slice UTF-16 units, which would split a surrogate pair. */
|
|
168
|
+
function previewOf(text) {
|
|
169
|
+
return Array.from(text ?? '').slice(0, PREVIEW_CODE_POINTS).join('');
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Fill the derived fields. `receipt_hash` is always computed here — a
|
|
173
|
+
* caller-supplied value would let the stored hash stop covering the receipt's
|
|
174
|
+
* identifying fields, which is the one property the hash exists to guarantee.
|
|
175
|
+
* A projection receipt's `unit` is likewise forced: the literal is the
|
|
176
|
+
* invariant T19/T26 filter on, not a caller parameter.
|
|
177
|
+
*/
|
|
178
|
+
function buildReceipt(r) {
|
|
179
|
+
const receipt = {
|
|
180
|
+
...r,
|
|
181
|
+
version: r.version ?? 1,
|
|
182
|
+
receipt_hash: receiptHash(r),
|
|
183
|
+
preview: previewOf(r.preview),
|
|
184
|
+
};
|
|
185
|
+
if (receipt.kind === 'projection')
|
|
186
|
+
receipt.unit = 'raw';
|
|
187
|
+
// Bound before `capArgs`: its `JSON.stringify` would throw on a tree deeper
|
|
188
|
+
// than the engine's recursion limit. The stored copy is the bounded one, so
|
|
189
|
+
// a reader can parse it and the depth cut is visible.
|
|
190
|
+
if (receipt.args !== undefined) {
|
|
191
|
+
const bounded = boundArgs(receipt.args);
|
|
192
|
+
receipt.args = bounded.value;
|
|
193
|
+
if (bounded.depthTruncated)
|
|
194
|
+
receipt.args_depth_truncated = true;
|
|
195
|
+
}
|
|
196
|
+
return capArgs(receipt);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Bound the `args` a receipt carries.
|
|
200
|
+
*
|
|
201
|
+
* The model writes the arguments, so an otherwise-valid plan can carry a
|
|
202
|
+
* multi-kilobyte blob into every receipt line. The hash is computed before this
|
|
203
|
+
* cap, over the depth-bounded tree `boundArgs` produced (see `receiptHash`), so
|
|
204
|
+
* identity is unaffected by this display cap; the stored copy is what an
|
|
205
|
+
* operator reads, and a `truncated_json` marker with `args_truncated` says so
|
|
206
|
+
* plainly rather than silently shortening a command.
|
|
207
|
+
*/
|
|
208
|
+
function capArgs(receipt) {
|
|
209
|
+
if (receipt.args === undefined)
|
|
210
|
+
return receipt;
|
|
211
|
+
const json = JSON.stringify(receipt.args);
|
|
212
|
+
if (json === undefined || Array.from(json).length <= MAX_RECEIPT_ARGS_CODE_POINTS)
|
|
213
|
+
return receipt;
|
|
214
|
+
return {
|
|
215
|
+
...receipt,
|
|
216
|
+
args: { truncated_json: capCodePoints(json, MAX_RECEIPT_ARGS_CODE_POINTS) },
|
|
217
|
+
args_truncated: true,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/** Cap a string by code point so a surrogate pair is never split. */
|
|
221
|
+
function capCodePoints(value, cap) {
|
|
222
|
+
const points = Array.from(value);
|
|
223
|
+
if (points.length <= cap)
|
|
224
|
+
return value;
|
|
225
|
+
return `${points.slice(0, Math.max(0, cap - 1)).join('')}…`;
|
|
226
|
+
}
|
|
227
|
+
/** One receipt as one newline-terminated JSON line. */
|
|
228
|
+
function serialiseReceipt(r) {
|
|
229
|
+
return `${JSON.stringify(buildReceipt(r))}\n`;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Encode a session id as one safe filesystem path segment.
|
|
233
|
+
*
|
|
234
|
+
* Only `[A-Za-z0-9_-]` survives; every other character becomes `%XX` (uppercase
|
|
235
|
+
* hex, one per UTF-8 byte). The mapping is reversible and keeps `/`, `\`, `:`
|
|
236
|
+
* and `.` out of the name, so a host-supplied id can never escape
|
|
237
|
+
* `<root>/receipts/` and always yields one writable shard. The receipt body
|
|
238
|
+
* keeps the original id; only the file name is encoded.
|
|
239
|
+
*/
|
|
240
|
+
function encodeSegment(sessionId) {
|
|
241
|
+
let encoded = '';
|
|
242
|
+
for (const char of sessionId) {
|
|
243
|
+
if (/^[A-Za-z0-9_-]$/.test(char)) {
|
|
244
|
+
encoded += char;
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
for (const byte of Buffer.from(char, 'utf8')) {
|
|
248
|
+
encoded += `%${byte.toString(16).toUpperCase().padStart(2, '0')}`;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return encoded;
|
|
252
|
+
}
|
|
253
|
+
/** Path of a shard from an already-encoded base name (used by scans over the directory). */
|
|
254
|
+
function shardFile(root, base, rotated) {
|
|
255
|
+
const name = rotated ? `${base}.1.jsonl` : `${base}.jsonl`;
|
|
256
|
+
return join(root, RECEIPTS_DIR, name);
|
|
257
|
+
}
|
|
258
|
+
/** `stat(dir).isDirectory()`-equivalent check without recursing into Node's `mkdir` spin. */
|
|
259
|
+
async function ensureDirectory(dir) {
|
|
260
|
+
try {
|
|
261
|
+
await mkdir(dir);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
const code = errorCode(error);
|
|
266
|
+
if (code === 'EEXIST') {
|
|
267
|
+
// `mkdir` reports EEXIST for a regular file too; only a directory is usable.
|
|
268
|
+
if ((await stat(dir)).isDirectory())
|
|
269
|
+
return;
|
|
270
|
+
throw error;
|
|
271
|
+
}
|
|
272
|
+
if (code !== 'ENOENT')
|
|
273
|
+
throw error;
|
|
274
|
+
}
|
|
275
|
+
// Node's own recursive `mkdir` can spin forever on filesystems that answer
|
|
276
|
+
// ENOENT for a child even though the parent exists (e.g. `/proc`, the
|
|
277
|
+
// fail-closed fixture's root). Walking up one segment at a time fails fast.
|
|
278
|
+
const parent = dirname(dir);
|
|
279
|
+
if (parent === dir)
|
|
280
|
+
throw new Error(`cannot create directory: ${dir}`);
|
|
281
|
+
await ensureDirectory(parent);
|
|
282
|
+
try {
|
|
283
|
+
await mkdir(dir);
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
if (errorCode(error) !== 'EEXIST' || !(await stat(dir)).isDirectory())
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* The complete-line prefix of `data`, computed without touching the file.
|
|
292
|
+
*
|
|
293
|
+
* A trailing line without `\n` was never finished, so it is not a receipt:
|
|
294
|
+
* `size` is the byte offset just past the last newline (zero when the first
|
|
295
|
+
* line is unfinished). This is pure because the read path must never write: it
|
|
296
|
+
* snapshots the file while a writer may be appending, and truncating from that
|
|
297
|
+
* snapshot can cut a line the writer just completed. Only `stateFor` (the write
|
|
298
|
+
* path) acts on `size < data.length`.
|
|
299
|
+
*/
|
|
300
|
+
function completePrefix(data) {
|
|
301
|
+
const lastNewline = data.lastIndexOf(NEWLINE);
|
|
302
|
+
const size = lastNewline + 1;
|
|
303
|
+
return { valid: data.subarray(0, size), size };
|
|
304
|
+
}
|
|
305
|
+
/** Read a shard, ignoring an unfinished tail line without modifying the file. */
|
|
306
|
+
async function readShard(path) {
|
|
307
|
+
let data;
|
|
308
|
+
try {
|
|
309
|
+
data = await readFile(path);
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
// ENOTDIR is the file-valued `$DSH_HOME` shape: the shard cannot exist.
|
|
313
|
+
const code = errorCode(error);
|
|
314
|
+
if (code === 'ENOENT' || code === 'ENOTDIR')
|
|
315
|
+
return [];
|
|
316
|
+
throw error;
|
|
317
|
+
}
|
|
318
|
+
const { valid } = completePrefix(data);
|
|
319
|
+
const receipts = [];
|
|
320
|
+
for (const line of valid.toString('utf8').split('\n')) {
|
|
321
|
+
if (line === '')
|
|
322
|
+
continue;
|
|
323
|
+
let parsed;
|
|
324
|
+
try {
|
|
325
|
+
parsed = JSON.parse(line);
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
continue; // Corrupt middle line: skip it, do not fail the whole scan.
|
|
329
|
+
}
|
|
330
|
+
if (isReceipt(parsed))
|
|
331
|
+
receipts.push(parsed);
|
|
332
|
+
}
|
|
333
|
+
return receipts;
|
|
334
|
+
}
|
|
335
|
+
function isReceipt(value) {
|
|
336
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
337
|
+
}
|
|
338
|
+
function errorCode(error) {
|
|
339
|
+
return typeof error === 'object' && error !== null && 'code' in error
|
|
340
|
+
? error.code
|
|
341
|
+
: undefined;
|
|
342
|
+
}
|
|
343
|
+
function countNewlines(data) {
|
|
344
|
+
let lines = 0;
|
|
345
|
+
for (const byte of data) {
|
|
346
|
+
if (byte === NEWLINE)
|
|
347
|
+
lines += 1;
|
|
348
|
+
}
|
|
349
|
+
return lines;
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Append-only, crash-safe, bounded store of receipts per session.
|
|
353
|
+
*
|
|
354
|
+
* See the module docs for the write and read paths. Concurrent method calls on
|
|
355
|
+
* one instance are safe: writes are serialised through an internal promise
|
|
356
|
+
* chain. The store is single-writer per root by design — rotation counters are
|
|
357
|
+
* per instance, so two instances (or processes) writing one root can exceed the
|
|
358
|
+
* shard caps and lose receipts. The plugin owns exactly one store per root.
|
|
359
|
+
*/
|
|
360
|
+
export class ReceiptStore {
|
|
361
|
+
root;
|
|
362
|
+
fsyncEveryLines;
|
|
363
|
+
maxBytes;
|
|
364
|
+
maxLines;
|
|
365
|
+
/** *Consecutive* write failures; reset by any successful append. */
|
|
366
|
+
failureCount = 0;
|
|
367
|
+
/**
|
|
368
|
+
* Shard reads skipped by whole-store scans since the last scan that read
|
|
369
|
+
* every shard (EACCES, EISDIR, …). A fully readable scan resets it to zero,
|
|
370
|
+
* so a repaired shard drops `ERR_RECEIPTS_UNREADABLE` instead of pinning it
|
|
371
|
+
* for the life of the process. Named scans neither count here nor reset it:
|
|
372
|
+
* they do not cover the shard set status reports.
|
|
373
|
+
*/
|
|
374
|
+
readFailureCount = 0;
|
|
375
|
+
/** Whether the most recent whole-store scan hit {@link MAX_SCAN_SHARDS}. */
|
|
376
|
+
scanCapHit = false;
|
|
377
|
+
linesSinceSync = 0;
|
|
378
|
+
pending = [];
|
|
379
|
+
states = new Map();
|
|
380
|
+
dirty = new Set();
|
|
381
|
+
disposed = new Set();
|
|
382
|
+
disposedOrder = [];
|
|
383
|
+
dirReady = false;
|
|
384
|
+
/** Writes run one at a time so rotation and buffer replay cannot interleave. */
|
|
385
|
+
chain = Promise.resolve();
|
|
386
|
+
constructor(root, opts = {}) {
|
|
387
|
+
this.root = root;
|
|
388
|
+
this.fsyncEveryLines = opts.fsyncEveryLines ?? DEFAULT_FSYNC_EVERY_LINES;
|
|
389
|
+
this.maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
390
|
+
this.maxLines = opts.maxLines ?? DEFAULT_MAX_LINES;
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Append one receipt. Never rejects: a serialisation or write failure is
|
|
394
|
+
* counted and buffered. The session id is encoded only for the shard name;
|
|
395
|
+
* the stored receipt keeps the original id.
|
|
396
|
+
*/
|
|
397
|
+
append(receipt) {
|
|
398
|
+
const run = this.chain.then(() => this.appendNow(receipt));
|
|
399
|
+
const safe = run.then(() => undefined, () => undefined);
|
|
400
|
+
this.chain = safe;
|
|
401
|
+
return safe;
|
|
402
|
+
}
|
|
403
|
+
/** Force the buffered lines and the current shards to disk (called at turn end). */
|
|
404
|
+
flush() {
|
|
405
|
+
// Queued like an append: a flush racing an append must not replay the same
|
|
406
|
+
// buffered line twice.
|
|
407
|
+
const run = this.chain.then(() => this.flushNow());
|
|
408
|
+
const safe = run.then(() => undefined, () => undefined);
|
|
409
|
+
this.chain = safe;
|
|
410
|
+
return safe;
|
|
411
|
+
}
|
|
412
|
+
/** Number of receipts readable for `sessionId` (`.1` plus current). */
|
|
413
|
+
async count(sessionId) {
|
|
414
|
+
let count = 0;
|
|
415
|
+
for await (const _ of this.scan(sessionId))
|
|
416
|
+
count += 1;
|
|
417
|
+
return count;
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Shard reads that the most recent whole-store scan skipped.
|
|
421
|
+
*
|
|
422
|
+
* A single unreadable shard (EACCES, a directory named `x.jsonl`, …) must
|
|
423
|
+
* not abort the whole scan: `boost status` would turn that into
|
|
424
|
+
* `ERR_BOOST_INTERNAL` for the entire ledger. Scans skip the shard and count
|
|
425
|
+
* it here so status can report a stable degraded code.
|
|
426
|
+
*
|
|
427
|
+
* The count describes the skipped reads **since the last whole-store scan
|
|
428
|
+
* that read every shard**, not the process lifetime: a fully readable scan
|
|
429
|
+
* resets it, so repairing the shard and re-running `status` drops the code,
|
|
430
|
+
* as the README instructs. A named `scan(sessionId)`/`count(sessionId)`
|
|
431
|
+
* neither contributes to nor clears it, because it does not cover the shard
|
|
432
|
+
* set the status total claims.
|
|
433
|
+
*/
|
|
434
|
+
readErrors() {
|
|
435
|
+
return this.readFailureCount;
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Whether the most recent whole-store scan stopped at {@link MAX_SCAN_SHARDS}.
|
|
439
|
+
*
|
|
440
|
+
* `status` reports a total over the shards it read, so a capped scan is a
|
|
441
|
+
* lower bound, not a total. The flag is per scan (false on the next full
|
|
442
|
+
* scan that fits), which lets the operator see the condition clear; the
|
|
443
|
+
* numeric totals stay unchanged and under-count, the safe direction.
|
|
444
|
+
*/
|
|
445
|
+
scanTruncated() {
|
|
446
|
+
return this.scanCapHit;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Yield readable receipts, oldest first: `.1` before the current shard.
|
|
450
|
+
*
|
|
451
|
+
* The optional `sessionId` is encoded to its shard name; the receipts keep
|
|
452
|
+
* their original `sessionId`. With no argument, every shard under `receipts/`
|
|
453
|
+
* is visited (encoded shard names sorted for determinism, capped at
|
|
454
|
+
* {@link MAX_SCAN_SHARDS}), and each receipt supplies its own id from its
|
|
455
|
+
* body. Unparseable middle lines are skipped; an unfinished trailing line is
|
|
456
|
+
* ignored by the read and only the next write repairs it.
|
|
457
|
+
*
|
|
458
|
+
* A whole-store scan also maintains the two status readings: a shard it
|
|
459
|
+
* cannot read increments {@link readErrors} (a fully readable scan resets
|
|
460
|
+
* it), and more shards than the cap sets {@link scanTruncated}. Named scans
|
|
461
|
+
* touch neither, because they are not the population `status` totals.
|
|
462
|
+
*/
|
|
463
|
+
async *scan(sessionId) {
|
|
464
|
+
await this.chain;
|
|
465
|
+
let shards;
|
|
466
|
+
let readFailed = false;
|
|
467
|
+
const wholeStore = sessionId === undefined;
|
|
468
|
+
if (wholeStore) {
|
|
469
|
+
this.scanCapHit = false;
|
|
470
|
+
try {
|
|
471
|
+
const names = await this.listShardNames();
|
|
472
|
+
this.scanCapHit = names.length > MAX_SCAN_SHARDS;
|
|
473
|
+
// Bounded status scan; named scans below still address any one shard.
|
|
474
|
+
shards = names.slice(0, MAX_SCAN_SHARDS);
|
|
475
|
+
}
|
|
476
|
+
catch {
|
|
477
|
+
// An unreadable receipts directory is a degraded status answer, not a
|
|
478
|
+
// thrown tool error. Count it and yield nothing.
|
|
479
|
+
this.readFailureCount += 1;
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
else {
|
|
484
|
+
shards = [encodeSegment(sessionId)];
|
|
485
|
+
}
|
|
486
|
+
for (const base of shards) {
|
|
487
|
+
for (const rotated of [true, false]) {
|
|
488
|
+
let receipts;
|
|
489
|
+
try {
|
|
490
|
+
receipts = await readShard(shardFile(this.root, base, rotated));
|
|
491
|
+
}
|
|
492
|
+
catch {
|
|
493
|
+
this.readFailureCount += 1;
|
|
494
|
+
readFailed = true;
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
for (const receipt of receipts) {
|
|
498
|
+
yield receipt;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
// A completed whole-store scan that read every shard clears the reading, so
|
|
503
|
+
// repairing the shard and re-running `status` drops ERR_RECEIPTS_UNREADABLE.
|
|
504
|
+
if (wholeStore && !readFailed)
|
|
505
|
+
this.readFailureCount = 0;
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Forget one session's shard state and silence later receipts for it.
|
|
509
|
+
*
|
|
510
|
+
* Called from the plugin's `session/disposed` handler after `flush()`. It
|
|
511
|
+
* evicts the cached counters, forgets any buffered lines for that session
|
|
512
|
+
* **only when no write failure is outstanding**, and remembers the id
|
|
513
|
+
* (bounded FIFO) so a hook that lands after disposal records nothing instead
|
|
514
|
+
* of recreating state nothing will ever clear.
|
|
515
|
+
*
|
|
516
|
+
* The conditional drop is the "flush before eviction" promise: a successful
|
|
517
|
+
* flush has already drained `pending`, so the loop is a no-op; a failed one
|
|
518
|
+
* leaves the session's lines there, and they are the only copy of that
|
|
519
|
+
* evidence. They stay queued for the next flush — the buffer is bounded by
|
|
520
|
+
* {@link MAX_PENDING_LINES} and is emptied once {@link stopped} is reached,
|
|
521
|
+
* so retaining them cannot grow without limit.
|
|
522
|
+
*/
|
|
523
|
+
disposeSession(sessionId) {
|
|
524
|
+
this.states.delete(sessionId);
|
|
525
|
+
this.dirty.delete(sessionId);
|
|
526
|
+
if (this.failureCount === 0) {
|
|
527
|
+
for (let i = this.pending.length - 1; i >= 0; i -= 1) {
|
|
528
|
+
if (this.pending[i]?.sessionId === sessionId)
|
|
529
|
+
this.pending.splice(i, 1);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
if (this.disposed.has(sessionId))
|
|
533
|
+
return;
|
|
534
|
+
this.disposed.add(sessionId);
|
|
535
|
+
this.disposedOrder.push(sessionId);
|
|
536
|
+
while (this.disposedOrder.length > MAX_DISPOSED_SESSIONS) {
|
|
537
|
+
const oldest = this.disposedOrder.shift();
|
|
538
|
+
if (oldest !== undefined)
|
|
539
|
+
this.disposed.delete(oldest);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
/** Consecutive write failures; a successful append resets it to zero. */
|
|
543
|
+
failures() {
|
|
544
|
+
return this.failureCount;
|
|
545
|
+
}
|
|
546
|
+
/** Fail-closed gate: true after {@link STOPPED_AFTER_FAILURES} consecutive write failures. */
|
|
547
|
+
stopped() {
|
|
548
|
+
return this.failureCount >= STOPPED_AFTER_FAILURES;
|
|
549
|
+
}
|
|
550
|
+
async appendNow(receipt) {
|
|
551
|
+
// A late hook on a disposed session is expected to run; it must record
|
|
552
|
+
// nothing, not buffer a line nothing will ever write.
|
|
553
|
+
if (this.disposed.has(receipt.sessionId))
|
|
554
|
+
return;
|
|
555
|
+
let entry;
|
|
556
|
+
try {
|
|
557
|
+
entry = { sessionId: receipt.sessionId, line: serialiseReceipt(receipt) };
|
|
558
|
+
}
|
|
559
|
+
catch {
|
|
560
|
+
this.recordFailure();
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
try {
|
|
564
|
+
await this.drainPending();
|
|
565
|
+
await this.writeLine(entry);
|
|
566
|
+
}
|
|
567
|
+
catch {
|
|
568
|
+
// Invalidating the cached counters makes the next attempt re-read the
|
|
569
|
+
// shard, which also cuts a partial line this failed write may have left.
|
|
570
|
+
// Re-checking the directory lets the store recover if it disappeared.
|
|
571
|
+
this.states.delete(entry.sessionId);
|
|
572
|
+
this.dirReady = false;
|
|
573
|
+
if (!this.stopped() && this.pending.length < MAX_PENDING_LINES)
|
|
574
|
+
this.pending.push(entry);
|
|
575
|
+
this.recordFailure();
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
this.failureCount = 0;
|
|
579
|
+
try {
|
|
580
|
+
await this.syncIfDue();
|
|
581
|
+
}
|
|
582
|
+
catch {
|
|
583
|
+
// The line is written, so it is not buffered again — but a failed fsync
|
|
584
|
+
// still means the write path is unhealthy.
|
|
585
|
+
this.failureCount += 1;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
/** The serialised body of {@link flush}; a failure is counted, never thrown. */
|
|
589
|
+
async flushNow() {
|
|
590
|
+
try {
|
|
591
|
+
await this.drainPending();
|
|
592
|
+
await this.syncDirty();
|
|
593
|
+
}
|
|
594
|
+
catch {
|
|
595
|
+
this.recordFailure();
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Count one write failure and enforce the buffer bound.
|
|
600
|
+
*
|
|
601
|
+
* Once the store is `stopped()` (three consecutive failures) it is
|
|
602
|
+
* fail-closed: appends are still attempted so a recovered disk can reset the
|
|
603
|
+
* count, but the replay buffer is dropped instead of growing for as long as
|
|
604
|
+
* the outage lasts. Before that point `MAX_PENDING_LINES` is the hard cap.
|
|
605
|
+
*/
|
|
606
|
+
recordFailure() {
|
|
607
|
+
this.failureCount += 1;
|
|
608
|
+
if (this.stopped())
|
|
609
|
+
this.pending.length = 0;
|
|
610
|
+
}
|
|
611
|
+
/** Replay buffered lines in order; the head stays queued until its write succeeds. */
|
|
612
|
+
async drainPending() {
|
|
613
|
+
while (this.pending.length > 0) {
|
|
614
|
+
const entry = this.pending[0];
|
|
615
|
+
// `length > 0` guarantees a dense head (the buffer is only appended to
|
|
616
|
+
// and shifted from); the check narrows the type without a bang. Returning
|
|
617
|
+
// rather than spinning keeps an impossible sparse buffer from looping.
|
|
618
|
+
if (entry === undefined)
|
|
619
|
+
return;
|
|
620
|
+
await this.writeLine(entry);
|
|
621
|
+
this.pending.shift();
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
async writeLine(entry) {
|
|
625
|
+
await this.ensureDir();
|
|
626
|
+
const state = await this.stateFor(entry.sessionId);
|
|
627
|
+
await this.rotateIfFull(entry.sessionId, state);
|
|
628
|
+
await appendFile(this.shardPath(entry.sessionId, false), entry.line, 'utf8');
|
|
629
|
+
state.lines += 1;
|
|
630
|
+
state.bytes += Buffer.byteLength(entry.line, 'utf8');
|
|
631
|
+
this.linesSinceSync += 1;
|
|
632
|
+
this.dirty.add(entry.sessionId);
|
|
633
|
+
}
|
|
634
|
+
async ensureDir() {
|
|
635
|
+
if (this.dirReady)
|
|
636
|
+
return;
|
|
637
|
+
await ensureDirectory(join(this.root, RECEIPTS_DIR));
|
|
638
|
+
this.dirReady = true;
|
|
639
|
+
}
|
|
640
|
+
/** Cached line/byte counters, loaded (and repaired) from disk on first use. */
|
|
641
|
+
async stateFor(sessionId) {
|
|
642
|
+
const cached = this.states.get(sessionId);
|
|
643
|
+
if (cached !== undefined)
|
|
644
|
+
return cached;
|
|
645
|
+
const path = this.shardPath(sessionId, false);
|
|
646
|
+
let data;
|
|
647
|
+
try {
|
|
648
|
+
data = await readFile(path);
|
|
649
|
+
}
|
|
650
|
+
catch (error) {
|
|
651
|
+
if (errorCode(error) !== 'ENOENT')
|
|
652
|
+
throw error;
|
|
653
|
+
data = Buffer.alloc(0);
|
|
654
|
+
}
|
|
655
|
+
// A crash left a partial line; appending after it would make the next
|
|
656
|
+
// receipt unreadable too. The write path repairs it before counting and
|
|
657
|
+
// writing, and propagates a failed repair so this append fails instead.
|
|
658
|
+
const { valid, size } = completePrefix(data);
|
|
659
|
+
if (size < data.length)
|
|
660
|
+
await truncate(path, size);
|
|
661
|
+
const state = { lines: countNewlines(valid), bytes: size };
|
|
662
|
+
this.states.set(sessionId, state);
|
|
663
|
+
return state;
|
|
664
|
+
}
|
|
665
|
+
async rotateIfFull(sessionId, state) {
|
|
666
|
+
if (state.lines < this.maxLines && state.bytes < this.maxBytes)
|
|
667
|
+
return;
|
|
668
|
+
if (this.dirty.has(sessionId)) {
|
|
669
|
+
await this.syncSession(sessionId);
|
|
670
|
+
this.dirty.delete(sessionId);
|
|
671
|
+
}
|
|
672
|
+
try {
|
|
673
|
+
// Renaming over an existing `.1` is the whole retention policy: only the
|
|
674
|
+
// current shard and its predecessor are ever kept.
|
|
675
|
+
await rename(this.shardPath(sessionId, false), this.shardPath(sessionId, true));
|
|
676
|
+
}
|
|
677
|
+
catch (error) {
|
|
678
|
+
if (errorCode(error) !== 'ENOENT')
|
|
679
|
+
throw error; // already gone: nothing to rotate
|
|
680
|
+
}
|
|
681
|
+
state.lines = 0;
|
|
682
|
+
state.bytes = 0;
|
|
683
|
+
}
|
|
684
|
+
async syncIfDue() {
|
|
685
|
+
if (this.linesSinceSync < this.fsyncEveryLines)
|
|
686
|
+
return;
|
|
687
|
+
await this.syncDirty();
|
|
688
|
+
}
|
|
689
|
+
async syncDirty() {
|
|
690
|
+
for (const sessionId of this.dirty)
|
|
691
|
+
await this.syncSession(sessionId);
|
|
692
|
+
this.dirty.clear();
|
|
693
|
+
this.linesSinceSync = 0;
|
|
694
|
+
}
|
|
695
|
+
async syncSession(sessionId) {
|
|
696
|
+
const handle = await open(this.shardPath(sessionId, false), 'a');
|
|
697
|
+
try {
|
|
698
|
+
await handle.sync();
|
|
699
|
+
}
|
|
700
|
+
finally {
|
|
701
|
+
await handle.close();
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
shardPath(sessionId, rotated) {
|
|
705
|
+
return shardFile(this.root, encodeSegment(sessionId), rotated);
|
|
706
|
+
}
|
|
707
|
+
/** Encoded shard base names present under `receipts/`, sorted. */
|
|
708
|
+
async listShardNames() {
|
|
709
|
+
let entries;
|
|
710
|
+
try {
|
|
711
|
+
entries = await readdir(join(this.root, RECEIPTS_DIR));
|
|
712
|
+
}
|
|
713
|
+
catch (error) {
|
|
714
|
+
// ENOTDIR: `$DSH_HOME` (or the boost root) names a file, so `<root>/receipts`
|
|
715
|
+
// cannot be a directory. That is an empty ledger, not an internal failure;
|
|
716
|
+
// `boost status` must still answer (review I5).
|
|
717
|
+
const code = errorCode(error);
|
|
718
|
+
if (code === 'ENOENT' || code === 'ENOTDIR')
|
|
719
|
+
return [];
|
|
720
|
+
throw error;
|
|
721
|
+
}
|
|
722
|
+
const names = new Set();
|
|
723
|
+
for (const name of entries) {
|
|
724
|
+
if (name.endsWith(ROTATED_SUFFIX))
|
|
725
|
+
names.add(name.slice(0, -ROTATED_SUFFIX.length));
|
|
726
|
+
else if (name.endsWith(CURRENT_SUFFIX))
|
|
727
|
+
names.add(name.slice(0, -CURRENT_SUFFIX.length));
|
|
728
|
+
}
|
|
729
|
+
// The empty base is the empty sessionId's shard (`.jsonl`). Keep it: scan()
|
|
730
|
+
// must yield every receipt, and `scan('')` can address it.
|
|
731
|
+
return [...names].sort();
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
//# sourceMappingURL=store.js.map
|