@sorbetai/runtime 0.1.0-alpha.16 → 0.1.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{contracts-CFmBr0LI.d.ts → catalog-BePaQpKY.d.ts} +1311 -1012
- package/dist/index.d.ts +232 -119
- package/dist/index.js +5392 -12112
- package/dist/index.js.map +1 -1
- package/dist/node.d.ts +33 -5
- package/dist/node.js +2 -1877
- package/dist/units-DoehpsMM.js +12775 -0
- package/dist/units-DoehpsMM.js.map +1 -0
- package/package.json +7 -7
- package/dist/node.js.map +0 -1
- package/dist/observation-DyvvxD35.js +0 -2713
- package/dist/observation-DyvvxD35.js.map +0 -1
package/dist/node.js
CHANGED
|
@@ -1,1878 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import * as s from "@earendil-works/pi-schema";
|
|
3
|
-
import { compile, decode, decodeWith, objectProperties } from "@sorbetai/schema";
|
|
4
|
-
import { isEqual, isError, isPromise, isString, pick } from "es-toolkit";
|
|
5
|
-
import ensureError from "ensure-error";
|
|
6
|
-
import { mkdir, open, readFile, realpath, rename, rm, stat, truncate, unlink } from "node:fs/promises";
|
|
7
|
-
import { basename, dirname, join, resolve } from "node:path";
|
|
8
|
-
import { brotliCompress, brotliDecompress, constants } from "node:zlib";
|
|
9
|
-
import { promisify } from "node:util";
|
|
1
|
+
import { a as readSealedUnitManifest, c as unsealUnit, d as readJsonlJournal, f as acquireJournalWriterLease, i as isUnitSealed, l as JsonlJournalReader, n as JsonlUnitStore, o as sealUnit, r as discardIncompleteSeal, s as sealedUnitLayout, t as readUnitSessions, u as JsonlJournalStore } from "./units-DoehpsMM.js";
|
|
10
2
|
|
|
11
|
-
|
|
12
|
-
async function createJournalFile(filePath) {
|
|
13
|
-
let handle;
|
|
14
|
-
try {
|
|
15
|
-
handle = await open(filePath, "ax", 384);
|
|
16
|
-
} catch (cause) {
|
|
17
|
-
if (isError(cause) && Reflect.get(cause, "code") === "EEXIST") return;
|
|
18
|
-
throw cause;
|
|
19
|
-
}
|
|
20
|
-
await handle.close();
|
|
21
|
-
try {
|
|
22
|
-
await syncDirectory(dirname(filePath));
|
|
23
|
-
} catch (cause) {
|
|
24
|
-
await unlink(filePath);
|
|
25
|
-
throw cause;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
async function canonicalJournalPath(filePath) {
|
|
29
|
-
const absoluteFilePath = resolve(filePath);
|
|
30
|
-
try {
|
|
31
|
-
return await realpath(absoluteFilePath);
|
|
32
|
-
} catch (cause) {
|
|
33
|
-
if (!isError(cause) || Reflect.get(cause, "code") !== "ENOENT") throw cause;
|
|
34
|
-
return resolve(await realpath(dirname(absoluteFilePath)), basename(absoluteFilePath));
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
async function syncDirectory(directory) {
|
|
38
|
-
if (process.platform === "win32") return;
|
|
39
|
-
const handle = await open(directory, "r");
|
|
40
|
-
try {
|
|
41
|
-
await handle.sync();
|
|
42
|
-
} finally {
|
|
43
|
-
await handle.close();
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
//#endregion
|
|
48
|
-
//#region src/journal/writer-lease.ts
|
|
49
|
-
const SQLITE_BUSY = 5;
|
|
50
|
-
/**
|
|
51
|
-
* Excludes legacy writers while an offline importer reads their immutable prefix.
|
|
52
|
-
* Acquiring creates only the mutex file; it never repairs or creates the Journal.
|
|
53
|
-
* The parent directory must exist. Release after all source reads have completed.
|
|
54
|
-
*/
|
|
55
|
-
async function acquireJournalWriterLease(journalFilePath) {
|
|
56
|
-
journalFilePath = await canonicalJournalPath(journalFilePath);
|
|
57
|
-
const { DatabaseSync } = await import("node:sqlite");
|
|
58
|
-
const database = new DatabaseSync(`${journalFilePath}.writer-lock`, { timeout: 0 });
|
|
59
|
-
try {
|
|
60
|
-
database.exec("PRAGMA cache_size=0; PRAGMA journal_mode=MEMORY; BEGIN IMMEDIATE");
|
|
61
|
-
} catch (cause) {
|
|
62
|
-
database.close();
|
|
63
|
-
if (isError(cause) && Reflect.get(cause, "errcode") === SQLITE_BUSY) throw new JournalWriterBusyError(journalFilePath, { cause });
|
|
64
|
-
throw cause;
|
|
65
|
-
}
|
|
66
|
-
let released = false;
|
|
67
|
-
return { release: () => {
|
|
68
|
-
if (released) return;
|
|
69
|
-
database.close();
|
|
70
|
-
released = true;
|
|
71
|
-
} };
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
//#endregion
|
|
75
|
-
//#region src/journal/recovery-fact.ts
|
|
76
|
-
const HeaderKeys = [
|
|
77
|
-
"id",
|
|
78
|
-
"sessionId",
|
|
79
|
-
"sequence",
|
|
80
|
-
"cursor",
|
|
81
|
-
"recordedAt",
|
|
82
|
-
"kind",
|
|
83
|
-
"schemaVersion"
|
|
84
|
-
];
|
|
85
|
-
const ModelKinds = new Set([
|
|
86
|
-
"model.requested",
|
|
87
|
-
"model.response.committed",
|
|
88
|
-
"model.request.unknown"
|
|
89
|
-
]);
|
|
90
|
-
const EffectKinds = new Set([
|
|
91
|
-
"effect.requested",
|
|
92
|
-
"effect.succeeded",
|
|
93
|
-
"effect.failed",
|
|
94
|
-
"effect.unknown"
|
|
95
|
-
]);
|
|
96
|
-
const ModelInvocationKinds = new Set([
|
|
97
|
-
"model.invocation.requested",
|
|
98
|
-
"model.invocation.response.committed",
|
|
99
|
-
"model.invocation.unknown"
|
|
100
|
-
]);
|
|
101
|
-
const CompactionModelKinds = new Set([
|
|
102
|
-
"compaction.model.requested",
|
|
103
|
-
"compaction.model.response.committed",
|
|
104
|
-
"compaction.model.failed"
|
|
105
|
-
]);
|
|
106
|
-
const CorrelationId = s.str({ minLength: 1 });
|
|
107
|
-
const RecoveryFactSchema = s.union(JournalFactDefinitions.map((definition) => {
|
|
108
|
-
const { schema, kind } = definition;
|
|
109
|
-
const properties = { ...pick(objectProperties(schema), HeaderKeys) };
|
|
110
|
-
if (definition.scope === "session") properties.operationId = objectProperties(definition.schema).operationId;
|
|
111
|
-
else properties.runId = objectProperties(definition.schema).runId;
|
|
112
|
-
if (ModelKinds.has(kind)) properties.payload = s.obj({ modelRequestId: CorrelationId }, { open: true });
|
|
113
|
-
if (ModelInvocationKinds.has(kind)) properties.payload = s.obj({ requestId: CorrelationId }, { open: true });
|
|
114
|
-
if (EffectKinds.has(kind)) properties.payload = s.obj({ effectAttemptId: CorrelationId }, { open: true });
|
|
115
|
-
if (CompactionModelKinds.has(kind)) properties.payload = s.obj({ attemptId: CorrelationId }, { open: true });
|
|
116
|
-
return s.obj(properties, { open: true });
|
|
117
|
-
}));
|
|
118
|
-
const RecoveryFactValidator = compile(RecoveryFactSchema);
|
|
119
|
-
function decodeJournalRecoveryFact(value) {
|
|
120
|
-
const fact = RecoveryFactValidator.check(value) ? value : decode(RecoveryFactSchema, value);
|
|
121
|
-
const identity = {
|
|
122
|
-
id: fact.id,
|
|
123
|
-
sessionId: fact.sessionId,
|
|
124
|
-
sequence: fact.sequence,
|
|
125
|
-
cursor: fact.cursor,
|
|
126
|
-
recordedAt: fact.recordedAt,
|
|
127
|
-
schemaVersion: fact.schemaVersion
|
|
128
|
-
};
|
|
129
|
-
switch (fact.kind) {
|
|
130
|
-
case "model.invocation.requested":
|
|
131
|
-
case "model.invocation.response.committed":
|
|
132
|
-
case "model.invocation.unknown": return {
|
|
133
|
-
...identity,
|
|
134
|
-
kind: fact.kind,
|
|
135
|
-
operationId: fact.operationId,
|
|
136
|
-
payload: { requestId: fact.payload.requestId }
|
|
137
|
-
};
|
|
138
|
-
case "model.requested":
|
|
139
|
-
case "model.response.committed":
|
|
140
|
-
case "model.request.unknown": return {
|
|
141
|
-
...identity,
|
|
142
|
-
kind: fact.kind,
|
|
143
|
-
runId: fact.runId,
|
|
144
|
-
payload: { modelRequestId: fact.payload.modelRequestId }
|
|
145
|
-
};
|
|
146
|
-
case "effect.requested":
|
|
147
|
-
case "effect.succeeded":
|
|
148
|
-
case "effect.failed":
|
|
149
|
-
case "effect.unknown": return {
|
|
150
|
-
...identity,
|
|
151
|
-
kind: fact.kind,
|
|
152
|
-
runId: fact.runId,
|
|
153
|
-
payload: { effectAttemptId: fact.payload.effectAttemptId }
|
|
154
|
-
};
|
|
155
|
-
case "compaction.model.requested":
|
|
156
|
-
case "compaction.model.response.committed":
|
|
157
|
-
case "compaction.model.failed": return {
|
|
158
|
-
...identity,
|
|
159
|
-
kind: fact.kind,
|
|
160
|
-
operationId: fact.operationId,
|
|
161
|
-
payload: { attemptId: fact.payload.attemptId }
|
|
162
|
-
};
|
|
163
|
-
case "submission.accepted":
|
|
164
|
-
case "submission.applied":
|
|
165
|
-
case "submission.placed": return {
|
|
166
|
-
...identity,
|
|
167
|
-
kind: fact.kind,
|
|
168
|
-
operationId: fact.operationId
|
|
169
|
-
};
|
|
170
|
-
case "submission.withdrawn":
|
|
171
|
-
case "submission.settled": return {
|
|
172
|
-
...identity,
|
|
173
|
-
kind: fact.kind,
|
|
174
|
-
operationId: fact.operationId
|
|
175
|
-
};
|
|
176
|
-
case "permission.grant.revoked":
|
|
177
|
-
case "plugin.record.committed":
|
|
178
|
-
case "session.forked":
|
|
179
|
-
case "model.request.base.committed":
|
|
180
|
-
case "goal.created":
|
|
181
|
-
case "goal.status.changed":
|
|
182
|
-
case "goal.continuation.requested":
|
|
183
|
-
case "goal.removed":
|
|
184
|
-
case "session.behavior.changed":
|
|
185
|
-
case "compaction.requested":
|
|
186
|
-
case "compaction.committed":
|
|
187
|
-
case "compaction.failed":
|
|
188
|
-
case "compaction.skipped":
|
|
189
|
-
case "tool.operation.accepted":
|
|
190
|
-
case "tool.operation.approval.requested":
|
|
191
|
-
case "tool.operation.approval.resolved":
|
|
192
|
-
case "tool.operation.permission.decided":
|
|
193
|
-
case "tool.operation.permission.failed":
|
|
194
|
-
case "tool.operation.effect.requested":
|
|
195
|
-
case "tool.operation.effect.succeeded":
|
|
196
|
-
case "tool.operation.effect.failed":
|
|
197
|
-
case "tool.operation.effect.unknown":
|
|
198
|
-
case "tool.operation.terminal": return {
|
|
199
|
-
...identity,
|
|
200
|
-
kind: fact.kind,
|
|
201
|
-
operationId: fact.operationId
|
|
202
|
-
};
|
|
203
|
-
default: return {
|
|
204
|
-
...identity,
|
|
205
|
-
kind: fact.kind,
|
|
206
|
-
runId: fact.runId
|
|
207
|
-
};
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
function journalRecoveryIdentityBytes(fact) {
|
|
211
|
-
return [
|
|
212
|
-
fact.id,
|
|
213
|
-
fact.sessionId,
|
|
214
|
-
fact.cursor,
|
|
215
|
-
fact.recordedAt,
|
|
216
|
-
fact.runId,
|
|
217
|
-
fact.operationId,
|
|
218
|
-
fact.payload?.modelRequestId,
|
|
219
|
-
fact.payload?.requestId,
|
|
220
|
-
fact.payload?.effectAttemptId,
|
|
221
|
-
fact.payload?.attemptId
|
|
222
|
-
].reduce((bytes, value) => value === void 0 ? bytes : bytes + utf8ByteLength(value), 0);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
//#endregion
|
|
226
|
-
//#region src/journal/jsonl-store.ts
|
|
227
|
-
const PersistedEnvelopeValidator$1 = compile(PersistedJournalEnvelopeSchema);
|
|
228
|
-
const DEFAULT_INLINE_STRING_BYTES = 64 * 1024;
|
|
229
|
-
const JOURNAL_SCAN_BUFFER_BYTES = 1024 * 1024;
|
|
230
|
-
const InlineStringBytesSchema$1 = s.int({ minimum: 1 });
|
|
231
|
-
const JournalReadLimitSchema = s.int({
|
|
232
|
-
minimum: 1,
|
|
233
|
-
maximum: JOURNAL_READ_MAX_LIMIT
|
|
234
|
-
});
|
|
235
|
-
const PersistedJournalFrameSchema = s.obj({
|
|
236
|
-
storageVersion: s.lit(4),
|
|
237
|
-
facts: s.arr(JsonValueSchema, {
|
|
238
|
-
minItems: 1,
|
|
239
|
-
maxItems: JOURNAL_APPEND_BATCH_MAX_FACTS
|
|
240
|
-
})
|
|
241
|
-
}, { open: false });
|
|
242
|
-
const PersistedJournalFrameValidator = compile(PersistedJournalFrameSchema);
|
|
243
|
-
var JsonlSubscription = class {
|
|
244
|
-
#factCount;
|
|
245
|
-
#readFrame;
|
|
246
|
-
#onClose;
|
|
247
|
-
#nextIndex;
|
|
248
|
-
#decodedFrame;
|
|
249
|
-
#waiting;
|
|
250
|
-
#failure;
|
|
251
|
-
#nextPending = false;
|
|
252
|
-
#closed = false;
|
|
253
|
-
constructor(factCount, readFrame$1, nextIndex, onClose) {
|
|
254
|
-
this.#factCount = factCount;
|
|
255
|
-
this.#readFrame = readFrame$1;
|
|
256
|
-
this.#nextIndex = nextIndex;
|
|
257
|
-
this.#onClose = onClose;
|
|
258
|
-
}
|
|
259
|
-
notify() {
|
|
260
|
-
const waiting = this.#waiting;
|
|
261
|
-
if (waiting === void 0) return;
|
|
262
|
-
this.#waiting = void 0;
|
|
263
|
-
waiting.resolve();
|
|
264
|
-
}
|
|
265
|
-
fail(failure) {
|
|
266
|
-
if (this.#closed || this.#failure !== void 0) return;
|
|
267
|
-
this.#failure = failure;
|
|
268
|
-
const waiting = this.#waiting;
|
|
269
|
-
this.#waiting = void 0;
|
|
270
|
-
waiting?.reject(failure);
|
|
271
|
-
}
|
|
272
|
-
close() {
|
|
273
|
-
if (this.#closed) return;
|
|
274
|
-
this.#closed = true;
|
|
275
|
-
this.#onClose();
|
|
276
|
-
this.notify();
|
|
277
|
-
}
|
|
278
|
-
[Symbol.asyncIterator]() {
|
|
279
|
-
return {
|
|
280
|
-
next: async () => {
|
|
281
|
-
if (this.#nextPending) throw new Error("Journal subscription does not support concurrent next() calls");
|
|
282
|
-
this.#nextPending = true;
|
|
283
|
-
try {
|
|
284
|
-
return await this.#next();
|
|
285
|
-
} finally {
|
|
286
|
-
this.#nextPending = false;
|
|
287
|
-
}
|
|
288
|
-
},
|
|
289
|
-
return: async () => {
|
|
290
|
-
this.close();
|
|
291
|
-
return {
|
|
292
|
-
done: true,
|
|
293
|
-
value: void 0
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
|
-
};
|
|
297
|
-
}
|
|
298
|
-
async #next() {
|
|
299
|
-
while (true) {
|
|
300
|
-
if (this.#failure !== void 0) throw this.#failure;
|
|
301
|
-
if (this.#closed) return {
|
|
302
|
-
done: true,
|
|
303
|
-
value: void 0
|
|
304
|
-
};
|
|
305
|
-
if (this.#nextIndex < this.#factCount()) {
|
|
306
|
-
const decoded = this.#decodedFrame;
|
|
307
|
-
if (decoded === void 0 || this.#nextIndex < decoded.firstFactIndex || this.#nextIndex >= decoded.firstFactIndex + decoded.facts.length) this.#decodedFrame = await this.#readFrame(this.#nextIndex);
|
|
308
|
-
const frame = this.#decodedFrame;
|
|
309
|
-
const fact = frame?.facts[this.#nextIndex - frame.firstFactIndex];
|
|
310
|
-
if (fact === void 0) throw new JournalCorruptionError("Journal index points outside its frame");
|
|
311
|
-
this.#nextIndex += 1;
|
|
312
|
-
return {
|
|
313
|
-
done: false,
|
|
314
|
-
value: fact
|
|
315
|
-
};
|
|
316
|
-
}
|
|
317
|
-
await new Promise((resolve$1, reject) => {
|
|
318
|
-
this.#waiting = {
|
|
319
|
-
resolve: resolve$1,
|
|
320
|
-
reject
|
|
321
|
-
};
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
};
|
|
326
|
-
async function parsePersistedFact(value, lineNumber, factNumber, artifactStore) {
|
|
327
|
-
try {
|
|
328
|
-
return decode(JournalFactSchema, await decodeJournalEnvelope(value, artifactStore));
|
|
329
|
-
} catch (cause) {
|
|
330
|
-
throw new JournalCorruptionError(`Journal frame ${lineNumber} fact ${factNumber} does not match its schema`, { cause });
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
async function parseFrame(frame, lineNumber, artifactStore) {
|
|
334
|
-
return await Promise.all(frame.facts.map(async (fact, index) => await parsePersistedFact(fact, lineNumber, index + 1, artifactStore)));
|
|
335
|
-
}
|
|
336
|
-
var JournalLineSyntaxError = class extends Error {
|
|
337
|
-
constructor(lineNumber, cause) {
|
|
338
|
-
super(`Journal line ${lineNumber} is not valid JSON`, { cause });
|
|
339
|
-
this.name = "JournalLineSyntaxError";
|
|
340
|
-
}
|
|
341
|
-
};
|
|
342
|
-
function parseLineFrame(line, lineNumber) {
|
|
343
|
-
let encoded;
|
|
344
|
-
try {
|
|
345
|
-
encoded = JSON.parse(line);
|
|
346
|
-
} catch (cause) {
|
|
347
|
-
throw new JournalLineSyntaxError(lineNumber, isError(cause) ? cause : new Error(String(cause)));
|
|
348
|
-
}
|
|
349
|
-
let frame;
|
|
350
|
-
try {
|
|
351
|
-
frame = PersistedJournalFrameValidator.check(encoded) ? encoded : decode(PersistedJournalFrameSchema, encoded);
|
|
352
|
-
} catch (cause) {
|
|
353
|
-
throw new JournalCorruptionError(`Journal line ${lineNumber} uses an unsupported Journal format; original file was preserved`, { cause });
|
|
354
|
-
}
|
|
355
|
-
return frame;
|
|
356
|
-
}
|
|
357
|
-
async function parseLine(line, lineNumber, artifactStore) {
|
|
358
|
-
return parseFrame(parseLineFrame(line, lineNumber), lineNumber, artifactStore);
|
|
359
|
-
}
|
|
360
|
-
const FactPositionSchema = s.obj({
|
|
361
|
-
schemaVersion: s.lit(JOURNAL_FACT_SCHEMA_VERSION),
|
|
362
|
-
id: s.str(),
|
|
363
|
-
cursor: JournalCursorSchema,
|
|
364
|
-
sessionId: s.str(),
|
|
365
|
-
sequence: s.int({ minimum: 1 })
|
|
366
|
-
}, { open: true });
|
|
367
|
-
const FactPositionValidator = compile(FactPositionSchema);
|
|
368
|
-
async function scanFrame(line, lineNumber, artifactStore, retainFrame, retainRecoveryFrame, recoveryLimits) {
|
|
369
|
-
const frame = parseLineFrame(line, lineNumber);
|
|
370
|
-
if (retainRecoveryFrame !== void 0) {
|
|
371
|
-
const facts = frame.facts.map((value, index) => {
|
|
372
|
-
try {
|
|
373
|
-
return decodeJournalRecoveryFact(PersistedEnvelopeValidator$1.check(value) ? value.fact : value);
|
|
374
|
-
} catch (cause) {
|
|
375
|
-
throw new JournalCorruptionError(`Journal frame ${lineNumber} fact ${index + 1} has unavailable lifecycle metadata`, { cause });
|
|
376
|
-
}
|
|
377
|
-
});
|
|
378
|
-
for (const fact of facts) if (recoveryLimits?.maxIdentityBytes !== void 0 && journalRecoveryIdentityBytes(fact) > recoveryLimits.maxIdentityBytes) throw new JournalReaderCapacityError(recoveryLimits.maxIndexedFacts, recoveryLimits.maxIndexedBytes, recoveryLimits.maxFrameBytes, recoveryLimits.maxIdentityBytes);
|
|
379
|
-
retainRecoveryFrame(facts);
|
|
380
|
-
return facts;
|
|
381
|
-
}
|
|
382
|
-
if (retainFrame !== void 0) {
|
|
383
|
-
const facts = await parseFrame(frame, lineNumber, artifactStore);
|
|
384
|
-
retainFrame(facts);
|
|
385
|
-
return facts;
|
|
386
|
-
}
|
|
387
|
-
return frame.facts.map((value) => {
|
|
388
|
-
const raw = PersistedEnvelopeValidator$1.check(value) ? value.fact : value;
|
|
389
|
-
return FactPositionValidator.check(raw) ? raw : decode(FactPositionSchema, raw);
|
|
390
|
-
});
|
|
391
|
-
}
|
|
392
|
-
function indexFrame(facts, offset, length, lineNumber, journal, limits) {
|
|
393
|
-
const frameIndex = journal.frames.length;
|
|
394
|
-
const firstFactIndex = journal.factLocators.length;
|
|
395
|
-
if (limits !== void 0) journal.indexedBytes = (journal.indexedBytes ?? 0) + 64;
|
|
396
|
-
for (const [factIndex, fact] of facts.entries()) {
|
|
397
|
-
if (limits !== void 0) {
|
|
398
|
-
journal.indexedBytes = (journal.indexedBytes ?? 0) + 64 + Buffer.byteLength(fact.cursor);
|
|
399
|
-
if (journal.indexedBytes > limits.maxIndexedBytes) throw new JournalReaderCapacityError(limits.maxIndexedFacts, limits.maxIndexedBytes);
|
|
400
|
-
}
|
|
401
|
-
const expectedSequence = journal.throughSequence + 1;
|
|
402
|
-
if (fact.sequence !== expectedSequence) throw new JournalCorruptionError(`Journal sequence ${fact.sequence} does not match expected sequence ${expectedSequence}`);
|
|
403
|
-
if (journal.sessionId === void 0) journal.sessionId = fact.sessionId;
|
|
404
|
-
else if (fact.sessionId !== journal.sessionId) throw new JournalCorruptionError("One JSONL Journal file cannot contain multiple sessions");
|
|
405
|
-
if (fact.cursor !== fact.id) throw new JournalCorruptionError("JSONL Journal cursor must equal its stable fact identity");
|
|
406
|
-
if (journal.cursorIndexes.has(fact.cursor)) throw new JournalCorruptionError(`Journal fact identity is duplicated: ${fact.id}`);
|
|
407
|
-
journal.cursorIndexes.set(fact.cursor, journal.factLocators.length);
|
|
408
|
-
journal.factLocators.push({
|
|
409
|
-
cursor: fact.cursor,
|
|
410
|
-
frameIndex,
|
|
411
|
-
factIndex
|
|
412
|
-
});
|
|
413
|
-
journal.throughSequence = fact.sequence;
|
|
414
|
-
journal.lastCursor = fact.cursor;
|
|
415
|
-
}
|
|
416
|
-
journal.frames.push({
|
|
417
|
-
offset,
|
|
418
|
-
length,
|
|
419
|
-
lineNumber,
|
|
420
|
-
firstFactIndex,
|
|
421
|
-
factCount: facts.length
|
|
422
|
-
});
|
|
423
|
-
journal.lastLineNumber = lineNumber;
|
|
424
|
-
}
|
|
425
|
-
async function scanJournal(filePath, artifactStore, retainFrame, synchronize = false, incremental, retainRecoveryFrame) {
|
|
426
|
-
const previous = incremental?.previous;
|
|
427
|
-
const journal = previous === void 0 ? {
|
|
428
|
-
frames: [],
|
|
429
|
-
factLocators: [],
|
|
430
|
-
cursorIndexes: /* @__PURE__ */ new Map(),
|
|
431
|
-
throughSequence: 0,
|
|
432
|
-
lastCursor: void 0,
|
|
433
|
-
lastLineNumber: 0,
|
|
434
|
-
sessionId: void 0,
|
|
435
|
-
needsSeparator: false,
|
|
436
|
-
fileSize: 0
|
|
437
|
-
} : {
|
|
438
|
-
...previous,
|
|
439
|
-
needsSeparator: false,
|
|
440
|
-
truncateAt: void 0
|
|
441
|
-
};
|
|
442
|
-
if (incremental?.indexRetention === "page") {
|
|
443
|
-
journal.frames = [];
|
|
444
|
-
journal.factLocators = [];
|
|
445
|
-
journal.cursorIndexes = /* @__PURE__ */ new Map();
|
|
446
|
-
journal.indexedBytes = 0;
|
|
447
|
-
}
|
|
448
|
-
const originalFrames = journal.frames.length;
|
|
449
|
-
const originalFacts = journal.factLocators.length;
|
|
450
|
-
incremental?.signal?.throwIfAborted();
|
|
451
|
-
const handle = await open(filePath, synchronize ? "r+" : "r");
|
|
452
|
-
const buffer = Buffer.allocUnsafe(JOURNAL_SCAN_BUFFER_BYTES);
|
|
453
|
-
const fragments = [];
|
|
454
|
-
let fragmentBytes = 0;
|
|
455
|
-
let position = previous?.fileSize ?? 0;
|
|
456
|
-
let lineOffset = position;
|
|
457
|
-
let lineNumber = (previous?.lastLineNumber ?? 0) + 1;
|
|
458
|
-
let separatorPending = previous?.needsSeparator ?? false;
|
|
459
|
-
let stoppedEarly = false;
|
|
460
|
-
try {
|
|
461
|
-
const { size: fileSize, dev, ino } = await handle.stat();
|
|
462
|
-
if (previous !== void 0 && (fileSize < previous.fileSize || previous.fileIdentity?.dev !== dev || previous.fileIdentity?.ino !== ino)) throw new JournalReaderResetRequiredError();
|
|
463
|
-
while (position < fileSize) {
|
|
464
|
-
incremental?.signal?.throwIfAborted();
|
|
465
|
-
const bytesRemaining = fileSize - position;
|
|
466
|
-
const readLength = Math.min(buffer.byteLength, bytesRemaining);
|
|
467
|
-
const { bytesRead } = await handle.read(buffer, 0, readLength, position);
|
|
468
|
-
if (bytesRead === 0) throw new JournalCorruptionError(`Journal ended at byte ${position} before its expected size of ${fileSize} bytes`);
|
|
469
|
-
if (separatorPending && buffer[0] !== 10) throw new JournalCorruptionError("A complete unterminated Journal frame requires a separator before appended frames");
|
|
470
|
-
let segmentStart = 0;
|
|
471
|
-
for (let index = 0; index < bytesRead; index += 1) {
|
|
472
|
-
if (buffer[index] !== 10) continue;
|
|
473
|
-
const segment = buffer.subarray(segmentStart, index);
|
|
474
|
-
const line = fragments.length === 0 ? segment : Buffer.concat([...fragments, segment], fragmentBytes + segment.byteLength);
|
|
475
|
-
if (separatorPending && line.byteLength === 0) {
|
|
476
|
-
separatorPending = false;
|
|
477
|
-
segmentStart = index + 1;
|
|
478
|
-
lineOffset = position + segmentStart;
|
|
479
|
-
continue;
|
|
480
|
-
}
|
|
481
|
-
separatorPending = false;
|
|
482
|
-
if (line.byteLength === 0) throw new JournalCorruptionError(`Journal line ${lineNumber} is empty`);
|
|
483
|
-
if (incremental?.maxFrameBytes !== void 0 && line.byteLength > incremental.maxFrameBytes) throw new JournalReaderCapacityError(incremental.maxIndexedFacts, incremental.maxIndexedBytes, incremental.maxFrameBytes);
|
|
484
|
-
if (line.byteLength > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new JournalCorruptionError(`Journal line ${lineNumber} exceeds the maximum frame size`);
|
|
485
|
-
let facts;
|
|
486
|
-
try {
|
|
487
|
-
incremental?.signal?.throwIfAborted();
|
|
488
|
-
facts = await scanFrame(line.toString("utf8"), lineNumber, artifactStore, retainFrame, retainRecoveryFrame, incremental);
|
|
489
|
-
} catch (cause) {
|
|
490
|
-
if (cause instanceof JournalLineSyntaxError) throw new JournalCorruptionError(cause.message, { cause: cause.cause });
|
|
491
|
-
throw cause;
|
|
492
|
-
}
|
|
493
|
-
incremental?.signal?.throwIfAborted();
|
|
494
|
-
if (incremental !== void 0 && journal.factLocators.length + facts.length > incremental.maxIndexedFacts) throw new JournalReaderCapacityError(incremental.maxIndexedFacts);
|
|
495
|
-
indexFrame(facts, lineOffset, line.byteLength, lineNumber, journal, incremental);
|
|
496
|
-
fragments.length = 0;
|
|
497
|
-
fragmentBytes = 0;
|
|
498
|
-
segmentStart = index + 1;
|
|
499
|
-
lineOffset = position + segmentStart;
|
|
500
|
-
lineNumber += 1;
|
|
501
|
-
if (incremental !== void 0 && journal.factLocators.length - originalFacts >= incremental.limit && lineOffset < fileSize) {
|
|
502
|
-
position = lineOffset;
|
|
503
|
-
stoppedEarly = true;
|
|
504
|
-
break;
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
if (stoppedEarly) break;
|
|
508
|
-
const trailing = buffer.subarray(segmentStart, bytesRead);
|
|
509
|
-
if (trailing.byteLength > 0) {
|
|
510
|
-
const copied = Buffer.from(trailing);
|
|
511
|
-
fragments.push(copied);
|
|
512
|
-
fragmentBytes += copied.byteLength;
|
|
513
|
-
if (incremental?.maxFrameBytes !== void 0 && fragmentBytes > incremental.maxFrameBytes) throw new JournalReaderCapacityError(incremental.maxIndexedFacts, incremental.maxIndexedBytes, incremental.maxFrameBytes);
|
|
514
|
-
if (fragmentBytes > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new JournalCorruptionError(`Journal line ${lineNumber} exceeds the maximum frame size`);
|
|
515
|
-
}
|
|
516
|
-
position += bytesRead;
|
|
517
|
-
}
|
|
518
|
-
let scanned = {
|
|
519
|
-
...journal,
|
|
520
|
-
needsSeparator: separatorPending,
|
|
521
|
-
fileSize: position,
|
|
522
|
-
observedFileSize: fileSize,
|
|
523
|
-
hasMore: stoppedEarly,
|
|
524
|
-
fileIdentity: {
|
|
525
|
-
dev,
|
|
526
|
-
ino
|
|
527
|
-
}
|
|
528
|
-
};
|
|
529
|
-
if (fragmentBytes > 0) {
|
|
530
|
-
incremental?.signal?.throwIfAborted();
|
|
531
|
-
const trailing = Buffer.concat(fragments, fragmentBytes);
|
|
532
|
-
try {
|
|
533
|
-
const facts = await scanFrame(trailing.toString("utf8"), lineNumber, artifactStore, retainFrame, retainRecoveryFrame, incremental);
|
|
534
|
-
if (incremental !== void 0 && journal.factLocators.length + facts.length > incremental.maxIndexedFacts) throw new JournalReaderCapacityError(incremental.maxIndexedFacts);
|
|
535
|
-
indexFrame(facts, lineOffset, trailing.byteLength, lineNumber, journal, incremental);
|
|
536
|
-
scanned = {
|
|
537
|
-
...scanned,
|
|
538
|
-
indexedBytes: journal.indexedBytes,
|
|
539
|
-
throughSequence: journal.throughSequence,
|
|
540
|
-
lastCursor: journal.lastCursor,
|
|
541
|
-
lastLineNumber: journal.lastLineNumber,
|
|
542
|
-
needsSeparator: true,
|
|
543
|
-
fileSize: position
|
|
544
|
-
};
|
|
545
|
-
} catch (cause) {
|
|
546
|
-
if (!(cause instanceof JournalLineSyntaxError)) throw cause;
|
|
547
|
-
scanned = {
|
|
548
|
-
...scanned,
|
|
549
|
-
needsSeparator: false,
|
|
550
|
-
fileSize: lineOffset,
|
|
551
|
-
truncateAt: lineOffset
|
|
552
|
-
};
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
if (synchronize) await handle.sync();
|
|
556
|
-
incremental?.signal?.throwIfAborted();
|
|
557
|
-
return scanned;
|
|
558
|
-
} catch (cause) {
|
|
559
|
-
if (previous !== void 0) {
|
|
560
|
-
const removed = journal.factLocators.splice(originalFacts);
|
|
561
|
-
for (const fact of removed) journal.cursorIndexes.delete(fact.cursor);
|
|
562
|
-
journal.frames.splice(originalFrames);
|
|
563
|
-
}
|
|
564
|
-
throw cause;
|
|
565
|
-
} finally {
|
|
566
|
-
await handle.close();
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
/**
|
|
570
|
-
* Reads a synchronized complete-frame prefix without acquiring writer ownership or
|
|
571
|
-
* repairing a torn tail. By default, visibility does not imply durability.
|
|
572
|
-
* `sync` flushes the same open file after reading its fixed byte prefix and fails if
|
|
573
|
-
* flushing is unavailable. It requires write access but never changes file bytes.
|
|
574
|
-
* This establishes file durability, not acknowledgement by the source writer;
|
|
575
|
-
* directory durability and filesystem guarantees remain platform-dependent.
|
|
576
|
-
*/
|
|
577
|
-
async function readJsonlJournal(filePath, options = {}) {
|
|
578
|
-
const facts = [];
|
|
579
|
-
await scanJournal(filePath, options.artifactStore, (frame) => facts.push(...frame), options.durability === "sync");
|
|
580
|
-
return facts;
|
|
581
|
-
}
|
|
582
|
-
/**
|
|
583
|
-
* Incremental read-only observer of an append-only file. Complete historical frames
|
|
584
|
-
* must remain immutable. No writer lease, tail repair or execution recovery occurs.
|
|
585
|
-
* Atomic frames may exceed the requested fact limit by at most one frame. Call close()
|
|
586
|
-
* to release its bounded identity/offset index. Replacement/truncation requires a new
|
|
587
|
-
* reader; an incomplete tail remains unread until it forms a complete valid frame.
|
|
588
|
-
*/
|
|
589
|
-
var JsonlJournalReader = class JsonlJournalReader {
|
|
590
|
-
#filePath;
|
|
591
|
-
#options;
|
|
592
|
-
#maxIndexedFacts;
|
|
593
|
-
#maxIndexedBytes;
|
|
594
|
-
#scanned;
|
|
595
|
-
#reading;
|
|
596
|
-
#closed = false;
|
|
597
|
-
constructor(filePath, options) {
|
|
598
|
-
this.#filePath = resolve(filePath);
|
|
599
|
-
this.#options = options;
|
|
600
|
-
this.#maxIndexedFacts = decode(s.int({ minimum: 1 }), options.maxIndexedFacts ?? 1e5);
|
|
601
|
-
this.#maxIndexedBytes = decode(s.int({ minimum: 1 }), options.maxIndexedBytes ?? 16 * 1024 * 1024);
|
|
602
|
-
}
|
|
603
|
-
static async open(filePath, options = {}) {
|
|
604
|
-
await (await open(filePath, options.durability === "sync" ? "r+" : "r")).close();
|
|
605
|
-
return new JsonlJournalReader(filePath, options);
|
|
606
|
-
}
|
|
607
|
-
read(options = {}) {
|
|
608
|
-
if (this.#closed) return Promise.reject(/* @__PURE__ */ new Error("Read-only Journal reader is closed"));
|
|
609
|
-
if (this.#reading !== void 0) return Promise.reject(/* @__PURE__ */ new Error("Read-only Journal reader does not support concurrent reads"));
|
|
610
|
-
const limit = decode(JournalReadLimitSchema, options.limit ?? JOURNAL_READ_DEFAULT_LIMIT);
|
|
611
|
-
const reading = this.#read(limit).finally(() => {
|
|
612
|
-
this.#reading = void 0;
|
|
613
|
-
});
|
|
614
|
-
this.#reading = reading;
|
|
615
|
-
return reading;
|
|
616
|
-
}
|
|
617
|
-
/** Reads lifecycle identities without hydrating any externalized content. This
|
|
618
|
-
* validates frames, positions and required recovery IDs, not full payload health.
|
|
619
|
-
* Externalized correlation IDs are unavailable rather than guessed. The frame
|
|
620
|
-
* ceiling bounds inline JSON before decoding and preserves bytes/cursor on failure. */
|
|
621
|
-
readRecovery(options = {}) {
|
|
622
|
-
if (this.#closed) return Promise.reject(/* @__PURE__ */ new Error("Read-only Journal reader is closed"));
|
|
623
|
-
if (this.#reading !== void 0) return Promise.reject(/* @__PURE__ */ new Error("Read-only Journal reader does not support concurrent reads"));
|
|
624
|
-
const limit = decode(JournalReadLimitSchema, options.limit ?? JOURNAL_READ_DEFAULT_LIMIT);
|
|
625
|
-
const maxFrameBytes = decode(s.int({
|
|
626
|
-
minimum: 1,
|
|
627
|
-
maximum: JOURNAL_APPEND_BATCH_MAX_BYTES
|
|
628
|
-
}), options.maxFrameBytes ?? JOURNAL_APPEND_BATCH_MAX_BYTES);
|
|
629
|
-
const maxIdentityBytes = decode(s.int({
|
|
630
|
-
minimum: 1,
|
|
631
|
-
maximum: JOURNAL_APPEND_BATCH_MAX_BYTES
|
|
632
|
-
}), options.maxIdentityBytes ?? 1024);
|
|
633
|
-
const reading = this.#readRecovery(limit, maxFrameBytes, maxIdentityBytes, options.signal).finally(() => {
|
|
634
|
-
this.#reading = void 0;
|
|
635
|
-
});
|
|
636
|
-
this.#reading = reading;
|
|
637
|
-
return reading;
|
|
638
|
-
}
|
|
639
|
-
async #readRecovery(limit, maxFrameBytes, maxIdentityBytes, signal) {
|
|
640
|
-
const facts = [];
|
|
641
|
-
const scanned = await scanJournal(this.#filePath, void 0, void 0, this.#options.durability === "sync", {
|
|
642
|
-
previous: this.#scanned,
|
|
643
|
-
indexRetention: this.#options.indexRetention,
|
|
644
|
-
limit,
|
|
645
|
-
maxIndexedFacts: this.#maxIndexedFacts,
|
|
646
|
-
maxIndexedBytes: this.#maxIndexedBytes,
|
|
647
|
-
maxFrameBytes,
|
|
648
|
-
maxIdentityBytes,
|
|
649
|
-
signal
|
|
650
|
-
}, (frame) => facts.push(...frame));
|
|
651
|
-
this.#scanned = scanned;
|
|
652
|
-
return {
|
|
653
|
-
facts,
|
|
654
|
-
cursor: scanned.lastCursor,
|
|
655
|
-
hasMore: scanned.hasMore ?? false,
|
|
656
|
-
sessionId: scanned.sessionId,
|
|
657
|
-
throughSequence: scanned.throughSequence,
|
|
658
|
-
fileBytes: scanned.observedFileSize ?? scanned.fileSize,
|
|
659
|
-
indexedFacts: scanned.factLocators.length,
|
|
660
|
-
indexedBytes: scanned.indexedBytes ?? 0,
|
|
661
|
-
incompleteTail: scanned.truncateAt !== void 0,
|
|
662
|
-
durability: this.#options.durability ?? "observed"
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
async #read(limit) {
|
|
666
|
-
const facts = [];
|
|
667
|
-
const scanned = await scanJournal(this.#filePath, this.#options.artifactStore, (frame) => facts.push(...frame), this.#options.durability === "sync", {
|
|
668
|
-
previous: this.#scanned,
|
|
669
|
-
indexRetention: this.#options.indexRetention,
|
|
670
|
-
limit,
|
|
671
|
-
maxIndexedFacts: this.#maxIndexedFacts,
|
|
672
|
-
maxIndexedBytes: this.#maxIndexedBytes
|
|
673
|
-
});
|
|
674
|
-
this.#scanned = scanned;
|
|
675
|
-
return {
|
|
676
|
-
facts,
|
|
677
|
-
cursor: scanned.lastCursor,
|
|
678
|
-
hasMore: scanned.hasMore ?? false,
|
|
679
|
-
sessionId: scanned.sessionId,
|
|
680
|
-
throughSequence: scanned.throughSequence,
|
|
681
|
-
fileBytes: scanned.observedFileSize ?? scanned.fileSize,
|
|
682
|
-
indexedFacts: scanned.factLocators.length,
|
|
683
|
-
indexedBytes: scanned.indexedBytes ?? 0,
|
|
684
|
-
incompleteTail: scanned.truncateAt !== void 0,
|
|
685
|
-
durability: this.#options.durability ?? "observed"
|
|
686
|
-
};
|
|
687
|
-
}
|
|
688
|
-
async close() {
|
|
689
|
-
this.#closed = true;
|
|
690
|
-
await this.#reading?.catch(() => {});
|
|
691
|
-
this.#scanned = void 0;
|
|
692
|
-
}
|
|
693
|
-
};
|
|
694
|
-
/**
|
|
695
|
-
* File-backed, single-session Journal with durable append-before-apply semantics.
|
|
696
|
-
* The store holds exclusive writer ownership from open through close. Reopen
|
|
697
|
-
* validates physical frames and rebuilds an offset ledger; reads decode typed
|
|
698
|
-
* payloads from their source frames.
|
|
699
|
-
*/
|
|
700
|
-
var JsonlJournalStore = class JsonlJournalStore {
|
|
701
|
-
#observation;
|
|
702
|
-
#filePath;
|
|
703
|
-
#readHandle;
|
|
704
|
-
#appendHandle;
|
|
705
|
-
#frames;
|
|
706
|
-
#factLocators;
|
|
707
|
-
#cursorIndexes;
|
|
708
|
-
#subscriptions = /* @__PURE__ */ new Set();
|
|
709
|
-
#activeReads = /* @__PURE__ */ new Set();
|
|
710
|
-
#artifactStore;
|
|
711
|
-
#inlineStringBytes;
|
|
712
|
-
#writerLease;
|
|
713
|
-
#sessionId;
|
|
714
|
-
#tail = Promise.resolve();
|
|
715
|
-
#acceptingAppends = true;
|
|
716
|
-
#failure;
|
|
717
|
-
#closed = false;
|
|
718
|
-
#closing;
|
|
719
|
-
#needsSeparator;
|
|
720
|
-
#fileSize;
|
|
721
|
-
constructor(filePath, readHandle, appendHandle, writerLease, loaded, options) {
|
|
722
|
-
this.#observation = options.observation;
|
|
723
|
-
this.#filePath = filePath;
|
|
724
|
-
this.#readHandle = readHandle;
|
|
725
|
-
this.#appendHandle = appendHandle;
|
|
726
|
-
this.#writerLease = writerLease;
|
|
727
|
-
this.#frames = loaded.frames;
|
|
728
|
-
this.#factLocators = loaded.factLocators;
|
|
729
|
-
this.#cursorIndexes = loaded.cursorIndexes;
|
|
730
|
-
this.#sessionId = loaded.sessionId;
|
|
731
|
-
this.#needsSeparator = loaded.needsSeparator;
|
|
732
|
-
this.#fileSize = loaded.fileSize;
|
|
733
|
-
this.#artifactStore = options.artifactStore;
|
|
734
|
-
this.#inlineStringBytes = decode(InlineStringBytesSchema$1, options.inlineStringBytes ?? DEFAULT_INLINE_STRING_BYTES);
|
|
735
|
-
}
|
|
736
|
-
static async open(filePath, options = {}) {
|
|
737
|
-
const startedAt = options.observation === void 0 ? void 0 : performance.now();
|
|
738
|
-
const startedAtUnixMs = options.observation === void 0 ? void 0 : Date.now();
|
|
739
|
-
try {
|
|
740
|
-
const store = await JsonlJournalStore.#open(filePath, options);
|
|
741
|
-
if (options.observation !== void 0 && startedAt !== void 0) observeRuntime(() => options.observation?.journalReopened?.({
|
|
742
|
-
sessionId: store.sessionId,
|
|
743
|
-
backend: "jsonl",
|
|
744
|
-
outcome: "opened",
|
|
745
|
-
durationSeconds: (performance.now() - startedAt) / 1e3,
|
|
746
|
-
startedAtUnixMs,
|
|
747
|
-
endedAtUnixMs: Date.now(),
|
|
748
|
-
factCount: store.#factLocators.length,
|
|
749
|
-
fileBytes: store.#fileSize
|
|
750
|
-
}));
|
|
751
|
-
return store;
|
|
752
|
-
} catch (cause) {
|
|
753
|
-
if (options.observation !== void 0 && startedAt !== void 0) observeRuntime(() => options.observation?.journalReopened?.({
|
|
754
|
-
backend: "jsonl",
|
|
755
|
-
outcome: "failed",
|
|
756
|
-
durationSeconds: (performance.now() - startedAt) / 1e3,
|
|
757
|
-
startedAtUnixMs,
|
|
758
|
-
endedAtUnixMs: Date.now()
|
|
759
|
-
}));
|
|
760
|
-
throw cause;
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
static async #open(filePath, options) {
|
|
764
|
-
const inlineStringBytes = decode(InlineStringBytesSchema$1, options.inlineStringBytes ?? DEFAULT_INLINE_STRING_BYTES);
|
|
765
|
-
if (options.inlineStringBytes !== void 0 && options.artifactStore === void 0) throw new TypeError("An inline string threshold requires an Artifact Store");
|
|
766
|
-
const normalizedOptions = {
|
|
767
|
-
...options,
|
|
768
|
-
inlineStringBytes
|
|
769
|
-
};
|
|
770
|
-
const absoluteFilePath = resolve(filePath);
|
|
771
|
-
await mkdir(dirname(absoluteFilePath), { recursive: true });
|
|
772
|
-
const storagePath = await canonicalJournalPath(absoluteFilePath);
|
|
773
|
-
const writerLease = await acquireJournalWriterLease(storagePath);
|
|
774
|
-
try {
|
|
775
|
-
await createJournalFile(storagePath);
|
|
776
|
-
const loaded = await scanJournal(storagePath, options.artifactStore);
|
|
777
|
-
if (loaded.truncateAt !== void 0) await truncate(storagePath, loaded.truncateAt);
|
|
778
|
-
const readHandle = await open(storagePath, "r");
|
|
779
|
-
let appendHandle;
|
|
780
|
-
try {
|
|
781
|
-
appendHandle = await open(storagePath, "a");
|
|
782
|
-
} catch (cause) {
|
|
783
|
-
await readHandle.close();
|
|
784
|
-
throw cause;
|
|
785
|
-
}
|
|
786
|
-
return new JsonlJournalStore(storagePath, readHandle, appendHandle, writerLease, loaded, normalizedOptions);
|
|
787
|
-
} catch (cause) {
|
|
788
|
-
writerLease.release();
|
|
789
|
-
throw cause;
|
|
790
|
-
}
|
|
791
|
-
}
|
|
792
|
-
get filePath() {
|
|
793
|
-
return this.#filePath;
|
|
794
|
-
}
|
|
795
|
-
get sessionId() {
|
|
796
|
-
return this.#sessionId;
|
|
797
|
-
}
|
|
798
|
-
appendBatch(inputs, options) {
|
|
799
|
-
if (!this.#acceptingAppends) return Promise.reject(this.#unavailableError());
|
|
800
|
-
if (inputs.some(({ fact }) => isDurableSubmissionFactKind(fact.kind))) return Promise.reject(/* @__PURE__ */ new Error("Durable Submissions require a Unit store"));
|
|
801
|
-
return this.#serialize(async () => {
|
|
802
|
-
const startedAt = this.#observation === void 0 ? void 0 : performance.now();
|
|
803
|
-
const startedAtUnixMs = this.#observation === void 0 ? void 0 : Date.now();
|
|
804
|
-
let bytes;
|
|
805
|
-
try {
|
|
806
|
-
this.#assertOperational();
|
|
807
|
-
if (inputs.length > JOURNAL_APPEND_BATCH_MAX_FACTS) throw new RangeError(`Journal batch exceeds ${JOURNAL_APPEND_BATCH_MAX_FACTS} facts`);
|
|
808
|
-
const first = inputs[0].fact;
|
|
809
|
-
this.#assertSession(first.sessionId);
|
|
810
|
-
if (options.expectedSequence !== this.#factLocators.length) throw new JournalConflictError(options.expectedSequence, this.#factLocators.length);
|
|
811
|
-
for (const [index, { fact }] of inputs.entries()) if (fact.sequence !== this.#factLocators.length + index + 1) throw new JournalConflictError(fact.sequence - index - 1, this.#factLocators.length);
|
|
812
|
-
const serializeFrame = (facts) => `{"storageVersion":4,"facts":[${facts.join(",")}]}`;
|
|
813
|
-
let serialized = serializeFrame(inputs.map(({ json }) => json));
|
|
814
|
-
const artifacts = this.#artifactStore;
|
|
815
|
-
if (artifacts !== void 0) {
|
|
816
|
-
const payloads = Buffer.byteLength(serialized) > JOURNAL_APPEND_BATCH_MAX_BYTES;
|
|
817
|
-
const encode = (wholePayload) => Promise.all(inputs.map(async ({ json }) => {
|
|
818
|
-
if (!wholePayload && json.length < this.#inlineStringBytes / 3) return json;
|
|
819
|
-
const value = decode(s.map(JsonValueSchema), JSON.parse(json));
|
|
820
|
-
if (wholePayload) return JSON.stringify(await encodeJournalPayload(value, artifacts));
|
|
821
|
-
const encoded = encodeJournalFact(value, artifacts, this.#inlineStringBytes);
|
|
822
|
-
return isPromise(encoded) ? JSON.stringify(await encoded) : json;
|
|
823
|
-
}));
|
|
824
|
-
serialized = serializeFrame(await encode(payloads));
|
|
825
|
-
if (!payloads && Buffer.byteLength(serialized) > JOURNAL_APPEND_BATCH_MAX_BYTES) serialized = serializeFrame(await encode(true));
|
|
826
|
-
}
|
|
827
|
-
const serializedBytes = Buffer.byteLength(serialized);
|
|
828
|
-
bytes = serializedBytes;
|
|
829
|
-
if (serializedBytes > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new RangeError(`Journal frame is ${serializedBytes} bytes; maximum is ${JOURNAL_APPEND_BATCH_MAX_BYTES}`);
|
|
830
|
-
this.#bindSession(first.sessionId);
|
|
831
|
-
const prefix = this.#needsSeparator ? "\n" : "";
|
|
832
|
-
const frameOffset = this.#fileSize + Buffer.byteLength(prefix);
|
|
833
|
-
const encodedFrame = `${prefix}${serialized}\n`;
|
|
834
|
-
try {
|
|
835
|
-
await this.#appendHandle.writeFile(encodedFrame, "utf8");
|
|
836
|
-
await this.#appendHandle.sync();
|
|
837
|
-
} catch (cause) {
|
|
838
|
-
throw this.#fault(isError(cause) ? cause : new Error(String(cause)));
|
|
839
|
-
}
|
|
840
|
-
this.#needsSeparator = false;
|
|
841
|
-
const frameIndex = this.#frames.length;
|
|
842
|
-
const firstFactIndex = this.#factLocators.length;
|
|
843
|
-
for (const [factIndex, { fact }] of inputs.entries()) {
|
|
844
|
-
this.#cursorIndexes.set(fact.cursor, this.#factLocators.length);
|
|
845
|
-
this.#factLocators.push({
|
|
846
|
-
cursor: fact.cursor,
|
|
847
|
-
frameIndex,
|
|
848
|
-
factIndex
|
|
849
|
-
});
|
|
850
|
-
}
|
|
851
|
-
this.#frames.push({
|
|
852
|
-
offset: frameOffset,
|
|
853
|
-
length: serializedBytes,
|
|
854
|
-
lineNumber: frameIndex + 1,
|
|
855
|
-
firstFactIndex,
|
|
856
|
-
factCount: inputs.length
|
|
857
|
-
});
|
|
858
|
-
this.#fileSize += Buffer.byteLength(encodedFrame);
|
|
859
|
-
for (const subscription of this.#subscriptions) subscription.notify();
|
|
860
|
-
if (this.#observation !== void 0 && startedAt !== void 0) observeRuntime(() => this.#observation?.journalBatchCompleted?.({
|
|
861
|
-
sessionId: first.sessionId,
|
|
862
|
-
backend: "jsonl",
|
|
863
|
-
outcome: "committed",
|
|
864
|
-
durationSeconds: (performance.now() - startedAt) / 1e3,
|
|
865
|
-
startedAtUnixMs,
|
|
866
|
-
endedAtUnixMs: Date.now(),
|
|
867
|
-
factCount: inputs.length,
|
|
868
|
-
bytes,
|
|
869
|
-
throughSequence: this.#factLocators.length
|
|
870
|
-
}));
|
|
871
|
-
} catch (cause) {
|
|
872
|
-
if (this.#observation !== void 0 && startedAt !== void 0) observeRuntime(() => this.#observation?.journalBatchCompleted?.({
|
|
873
|
-
sessionId: inputs[0].fact.sessionId,
|
|
874
|
-
backend: "jsonl",
|
|
875
|
-
outcome: "failed",
|
|
876
|
-
durationSeconds: (performance.now() - startedAt) / 1e3,
|
|
877
|
-
startedAtUnixMs,
|
|
878
|
-
endedAtUnixMs: Date.now(),
|
|
879
|
-
factCount: inputs.length,
|
|
880
|
-
bytes
|
|
881
|
-
}));
|
|
882
|
-
throw cause;
|
|
883
|
-
}
|
|
884
|
-
});
|
|
885
|
-
}
|
|
886
|
-
append(input, options) {
|
|
887
|
-
return this.appendBatch([input], options);
|
|
888
|
-
}
|
|
889
|
-
async read(sessionId, options = {}) {
|
|
890
|
-
this.#assertOperational();
|
|
891
|
-
this.#assertSession(sessionId);
|
|
892
|
-
const limit = decode(JournalReadLimitSchema, options.limit ?? JOURNAL_READ_DEFAULT_LIMIT);
|
|
893
|
-
const start = this.#indexAfter(options.after);
|
|
894
|
-
const facts = await this.#trackRead(this.#readFacts(start, limit));
|
|
895
|
-
const cursor = facts.at(-1)?.cursor ?? options.after;
|
|
896
|
-
const page = {
|
|
897
|
-
facts,
|
|
898
|
-
hasMore: start + facts.length < this.#factLocators.length
|
|
899
|
-
};
|
|
900
|
-
return cursor === void 0 ? page : {
|
|
901
|
-
...page,
|
|
902
|
-
cursor
|
|
903
|
-
};
|
|
904
|
-
}
|
|
905
|
-
subscribe(sessionId, options = {}) {
|
|
906
|
-
this.#assertOperational();
|
|
907
|
-
this.#bindSession(sessionId);
|
|
908
|
-
const start = this.#indexAfter(options.after);
|
|
909
|
-
let subscription;
|
|
910
|
-
subscription = new JsonlSubscription(() => this.#factLocators.length, async (factIndex) => await this.#trackRead(this.#decodeFrameAt(factIndex)), start, () => {
|
|
911
|
-
this.#subscriptions.delete(subscription);
|
|
912
|
-
});
|
|
913
|
-
this.#subscriptions.add(subscription);
|
|
914
|
-
return subscription;
|
|
915
|
-
}
|
|
916
|
-
close() {
|
|
917
|
-
this.#closing ??= this.#close();
|
|
918
|
-
return this.#closing;
|
|
919
|
-
}
|
|
920
|
-
async #close() {
|
|
921
|
-
this.#acceptingAppends = false;
|
|
922
|
-
const failures = [];
|
|
923
|
-
try {
|
|
924
|
-
const tailResult = await Promise.allSettled([this.#tail]);
|
|
925
|
-
if (tailResult[0]?.status === "rejected") failures.push(tailResult[0].reason);
|
|
926
|
-
this.#closed = true;
|
|
927
|
-
for (const subscription of this.#subscriptions) try {
|
|
928
|
-
subscription.close();
|
|
929
|
-
} catch (cause) {
|
|
930
|
-
failures.push(cause);
|
|
931
|
-
}
|
|
932
|
-
const readResults = await Promise.allSettled([...this.#activeReads]);
|
|
933
|
-
for (const result of readResults) if (result.status === "rejected") failures.push(result.reason);
|
|
934
|
-
const handleResults = await Promise.allSettled([this.#readHandle.close(), this.#appendHandle.close()]);
|
|
935
|
-
for (const result of handleResults) if (result.status === "rejected") failures.push(result.reason);
|
|
936
|
-
} finally {
|
|
937
|
-
try {
|
|
938
|
-
this.#writerLease.release();
|
|
939
|
-
} catch (cause) {
|
|
940
|
-
failures.push(cause);
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
if (failures.length === 1) throw failures[0];
|
|
944
|
-
if (failures.length > 1) throw new AggregateError(failures, "Journal close failed");
|
|
945
|
-
}
|
|
946
|
-
async #readFacts(start, limit) {
|
|
947
|
-
const facts = [];
|
|
948
|
-
const end = Math.min(start + limit, this.#factLocators.length);
|
|
949
|
-
let index = start;
|
|
950
|
-
while (index < end) {
|
|
951
|
-
const frame = await this.#decodeFrameAt(index);
|
|
952
|
-
const localStart = index - frame.firstFactIndex;
|
|
953
|
-
const available = Math.min(frame.facts.length - localStart, end - index);
|
|
954
|
-
if (available <= 0) throw new JournalCorruptionError("Journal index points outside its frame");
|
|
955
|
-
facts.push(...frame.facts.slice(localStart, localStart + available));
|
|
956
|
-
index += available;
|
|
957
|
-
}
|
|
958
|
-
return facts;
|
|
959
|
-
}
|
|
960
|
-
async #decodeFrameAt(factIndex) {
|
|
961
|
-
const factLocator = this.#factLocators[factIndex];
|
|
962
|
-
if (factLocator === void 0) throw new JournalCorruptionError("Journal fact index is outside the durable ledger");
|
|
963
|
-
const frame = this.#frames[factLocator.frameIndex];
|
|
964
|
-
if (frame === void 0) throw new JournalCorruptionError("Journal frame index is outside the durable ledger");
|
|
965
|
-
const buffer = Buffer.allocUnsafe(frame.length);
|
|
966
|
-
let bytesRead = 0;
|
|
967
|
-
while (bytesRead < buffer.byteLength) {
|
|
968
|
-
const result = await this.#readHandle.read(buffer, bytesRead, buffer.byteLength - bytesRead, frame.offset + bytesRead);
|
|
969
|
-
if (result.bytesRead === 0) throw new JournalCorruptionError(`Journal frame ${frame.lineNumber} ended unexpectedly`);
|
|
970
|
-
bytesRead += result.bytesRead;
|
|
971
|
-
}
|
|
972
|
-
let facts;
|
|
973
|
-
try {
|
|
974
|
-
facts = await parseLine(buffer.toString("utf8"), frame.lineNumber, this.#artifactStore);
|
|
975
|
-
} catch (cause) {
|
|
976
|
-
if (cause instanceof JournalLineSyntaxError) throw new JournalCorruptionError(cause.message, { cause: cause.cause });
|
|
977
|
-
throw cause;
|
|
978
|
-
}
|
|
979
|
-
if (facts.length !== frame.factCount) throw new JournalCorruptionError(`Journal frame ${frame.lineNumber} fact count changed`);
|
|
980
|
-
for (const [index, fact] of facts.entries()) {
|
|
981
|
-
const locator = this.#factLocators[frame.firstFactIndex + index];
|
|
982
|
-
if (locator === void 0 || fact.cursor !== locator.cursor) throw new JournalCorruptionError(`Journal frame ${frame.lineNumber} identity changed`);
|
|
983
|
-
}
|
|
984
|
-
return {
|
|
985
|
-
firstFactIndex: frame.firstFactIndex,
|
|
986
|
-
facts
|
|
987
|
-
};
|
|
988
|
-
}
|
|
989
|
-
#trackRead(operation) {
|
|
990
|
-
this.#activeReads.add(operation);
|
|
991
|
-
operation.then(() => this.#activeReads.delete(operation), () => this.#activeReads.delete(operation));
|
|
992
|
-
return operation;
|
|
993
|
-
}
|
|
994
|
-
#serialize(operation) {
|
|
995
|
-
const result = this.#tail.then(operation);
|
|
996
|
-
this.#tail = result.then(() => void 0, () => void 0);
|
|
997
|
-
return result;
|
|
998
|
-
}
|
|
999
|
-
#bindSession(sessionId) {
|
|
1000
|
-
this.#assertOperational();
|
|
1001
|
-
if (this.#sessionId === void 0) this.#sessionId = sessionId;
|
|
1002
|
-
else this.#assertSession(sessionId);
|
|
1003
|
-
}
|
|
1004
|
-
#assertSession(sessionId) {
|
|
1005
|
-
if (this.#sessionId === void 0) return;
|
|
1006
|
-
if (this.#sessionId !== sessionId) throw new TypeError("Journal session does not belong to this JSONL file");
|
|
1007
|
-
}
|
|
1008
|
-
#assertOperational() {
|
|
1009
|
-
if (this.#closed) throw new Error("Journal store is closed");
|
|
1010
|
-
if (this.#failure !== void 0) throw this.#unavailableError();
|
|
1011
|
-
}
|
|
1012
|
-
#unavailableError() {
|
|
1013
|
-
if (this.#failure !== void 0) return new Error("Journal store is faulted; close and reopen it before continuing", { cause: this.#failure });
|
|
1014
|
-
return /* @__PURE__ */ new Error("Journal store is closed");
|
|
1015
|
-
}
|
|
1016
|
-
#fault(cause) {
|
|
1017
|
-
const failure = new JournalStoreFaultError("Journal write failed with an unknown durable outcome; close and reopen the Store", { cause });
|
|
1018
|
-
this.#failure = failure;
|
|
1019
|
-
this.#acceptingAppends = false;
|
|
1020
|
-
for (const subscription of this.#subscriptions) subscription.fail(failure);
|
|
1021
|
-
return failure;
|
|
1022
|
-
}
|
|
1023
|
-
#indexAfter(cursor) {
|
|
1024
|
-
if (cursor === void 0) return 0;
|
|
1025
|
-
const index = this.#cursorIndexes.get(cursor);
|
|
1026
|
-
if (index === void 0) throw new JournalCursorGapError(cursor);
|
|
1027
|
-
return index + 1;
|
|
1028
|
-
}
|
|
1029
|
-
};
|
|
1030
|
-
|
|
1031
|
-
//#endregion
|
|
1032
|
-
//#region src/journal/unit/frame.ts
|
|
1033
|
-
/**
|
|
1034
|
-
* Measured on a 24 MiB Unit: 8 KiB already deduplicates Tool previews, and 4 KiB additionally
|
|
1035
|
-
* captures the state messages repeated by every model request. Halving again gains under 2%
|
|
1036
|
-
* while roughly doubling the Artifact count, so this is the knee rather than the extreme.
|
|
1037
|
-
*/
|
|
1038
|
-
const DEFAULT_UNIT_INLINE_STRING_BYTES = 4 * 1024;
|
|
1039
|
-
const InlineStringBytesSchema = s.int({ minimum: 1 });
|
|
1040
|
-
const Utf8Decoder = new TextDecoder("utf-8", { fatal: true });
|
|
1041
|
-
const FrameIdentity = {
|
|
1042
|
-
format: s.lit("sorbet.unit"),
|
|
1043
|
-
unitId: s.str({ minLength: 1 }),
|
|
1044
|
-
epochId: s.str({ minLength: 1 }),
|
|
1045
|
-
commitId: s.str({ minLength: 1 }),
|
|
1046
|
-
firstSequence: s.int({ minimum: 1 })
|
|
1047
|
-
};
|
|
1048
|
-
const EncodedEntrySchema = s.obj({
|
|
1049
|
-
scope: s.union([s.lit("session"), s.lit("unit")]),
|
|
1050
|
-
fact: {}
|
|
1051
|
-
}, { open: true });
|
|
1052
|
-
const LogicalFrameProperties = {
|
|
1053
|
-
...FrameIdentity,
|
|
1054
|
-
storageVersion: s.lit(1),
|
|
1055
|
-
entries: s.arr(EncodedEntrySchema, { minItems: 1 })
|
|
1056
|
-
};
|
|
1057
|
-
const LogicalFrameSchema = s.obj(LogicalFrameProperties, { open: false });
|
|
1058
|
-
const InlineFrameSchema = s.obj({
|
|
1059
|
-
...LogicalFrameProperties,
|
|
1060
|
-
entries: s.arr(EncodedEntrySchema, {
|
|
1061
|
-
minItems: 1,
|
|
1062
|
-
maxItems: JOURNAL_APPEND_BATCH_MAX_FACTS
|
|
1063
|
-
})
|
|
1064
|
-
}, { open: false });
|
|
1065
|
-
const ReferenceFrameSchema = s.obj({
|
|
1066
|
-
...FrameIdentity,
|
|
1067
|
-
storageVersion: s.lit(2),
|
|
1068
|
-
transaction: s.obj({
|
|
1069
|
-
id: s.str({ pattern: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" }),
|
|
1070
|
-
count: s.int({ minimum: 1 }),
|
|
1071
|
-
byteLength: s.int({ minimum: 1 })
|
|
1072
|
-
}, { open: false })
|
|
1073
|
-
}, { open: false });
|
|
1074
|
-
const FrameValidator = compile(s.union([InlineFrameSchema, ReferenceFrameSchema]));
|
|
1075
|
-
const LogicalFrameValidator = compile(LogicalFrameSchema);
|
|
1076
|
-
/** Version 1 remains readable; only version 2 can refer to an oversized atomic batch. */
|
|
1077
|
-
function readFrame(input, transactions, inspection = false) {
|
|
1078
|
-
try {
|
|
1079
|
-
const frame = decodeWith(FrameValidator, JSON.parse(input));
|
|
1080
|
-
if (frame.storageVersion === 1) return frame;
|
|
1081
|
-
return readReferencedFrame(frame, transactions, inspection);
|
|
1082
|
-
} catch (cause) {
|
|
1083
|
-
if (cause instanceof UnitStoreError) throw cause;
|
|
1084
|
-
throw new UnitStoreError("unit_corrupt", "Cannot decode a unit frame", { cause });
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
async function readReferencedFrame(frame, transactions, inspection) {
|
|
1088
|
-
try {
|
|
1089
|
-
const { transaction, storageVersion: _version,...identity } = frame;
|
|
1090
|
-
if (inspection && transaction.byteLength > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new UnitStoreError("unit_inspection_capacity", "Transaction exceeds the diagnostic decoding budget");
|
|
1091
|
-
const text = await transactions.read(transaction.id, inspection ? JOURNAL_APPEND_BATCH_MAX_BYTES : void 0);
|
|
1092
|
-
const { entries, storageVersion: _bodyVersion,...bodyIdentity } = decodeWith(LogicalFrameValidator, JSON.parse(text));
|
|
1093
|
-
if (utf8ByteLength(text) !== transaction.byteLength || entries.length !== transaction.count || transaction.id !== identity.commitId || !isEqual(identity, bodyIdentity)) throw new UnitStoreError("unit_corrupt", "Transaction does not match its committed marker");
|
|
1094
|
-
return {
|
|
1095
|
-
...identity,
|
|
1096
|
-
storageVersion: 1,
|
|
1097
|
-
entries
|
|
1098
|
-
};
|
|
1099
|
-
} catch (cause) {
|
|
1100
|
-
if (cause instanceof UnitStoreError) throw cause;
|
|
1101
|
-
throw new UnitStoreError("unit_corrupt", "Cannot read a committed Unit transaction", { cause });
|
|
1102
|
-
}
|
|
1103
|
-
}
|
|
1104
|
-
const UnitFactValidator = compile(UnitFactSchema);
|
|
1105
|
-
/**
|
|
1106
|
-
* The staging writer validated every fact before its commit, so a complete committed frame
|
|
1107
|
-
* is trusted on read. Rechecking each fact against the full fact unions on every reopen
|
|
1108
|
-
* doubled decode time without finding anything the framing checks miss.
|
|
1109
|
-
*/
|
|
1110
|
-
const CommittedSessionFact = {};
|
|
1111
|
-
const CommittedUnitFact = {};
|
|
1112
|
-
const FactObjectValidator = compile(s.map(JsonValueSchema));
|
|
1113
|
-
const PersistedEnvelopeValidator = compile(PersistedJournalEnvelopeSchema);
|
|
1114
|
-
const UnitFactKindValidator = compile(s.obj({ kind: s.str() }, { open: true }));
|
|
1115
|
-
const SessionHeaderValidator = compile(s.obj({ sessionId: s.str() }, { open: true }));
|
|
1116
|
-
async function decodeUnitInspectionFrame(input, artifacts, transactions, sessionId, diagnostic = true, unitKinds) {
|
|
1117
|
-
const frame = await readFrame(Utf8Decoder.decode(input), transactions, diagnostic);
|
|
1118
|
-
const unitFacts = [];
|
|
1119
|
-
const sessionFacts = [];
|
|
1120
|
-
for (const entry of frame.entries) {
|
|
1121
|
-
const envelope = PersistedEnvelopeValidator.check(entry.fact) ? entry.fact : void 0;
|
|
1122
|
-
const raw = envelope === void 0 ? entry.fact : envelope.fact;
|
|
1123
|
-
if (entry.scope === "unit") {
|
|
1124
|
-
if (unitKinds !== void 0) {
|
|
1125
|
-
if (!UnitFactKindValidator.check(raw)) throw new UnitStoreError("unit_corrupt", "Unit fact has no kind");
|
|
1126
|
-
if (!unitKinds.has(raw.kind)) continue;
|
|
1127
|
-
}
|
|
1128
|
-
const fact = envelope === void 0 ? raw : await decodeJournalEnvelope(envelope, artifacts);
|
|
1129
|
-
unitFacts.push(UnitFactValidator.check(fact) ? fact : decodeWith(UnitFactValidator, fact));
|
|
1130
|
-
} else if (sessionId !== void 0) {
|
|
1131
|
-
if (!SessionHeaderValidator.check(raw)) throw new UnitStoreError("unit_corrupt", "Unit fact has no Session identity");
|
|
1132
|
-
if (raw.sessionId !== sessionId) continue;
|
|
1133
|
-
const fact = decodeJournalRecoveryFact(raw);
|
|
1134
|
-
sessionFacts.push(fact);
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
return {
|
|
1138
|
-
unitId: frame.unitId,
|
|
1139
|
-
firstSequence: frame.firstSequence,
|
|
1140
|
-
count: frame.entries.length,
|
|
1141
|
-
unitFacts,
|
|
1142
|
-
sessionFacts
|
|
1143
|
-
};
|
|
1144
|
-
}
|
|
1145
|
-
async function encodeUnitFrame(writes, options, transactions) {
|
|
1146
|
-
const first = writes[0]?.record;
|
|
1147
|
-
if (first === void 0) throw new RangeError("Unit batches require at least one fact");
|
|
1148
|
-
const header = JSON.stringify({
|
|
1149
|
-
format: "sorbet.unit",
|
|
1150
|
-
storageVersion: 1,
|
|
1151
|
-
unitId: first.unitId,
|
|
1152
|
-
epochId: first.epochId,
|
|
1153
|
-
commitId: first.commitId,
|
|
1154
|
-
firstSequence: first.unitSequence
|
|
1155
|
-
});
|
|
1156
|
-
const frame = (entries) => `${header.slice(0, -1)},"entries":[${entries.join(",")}]}`;
|
|
1157
|
-
let json = frame(writes.map(({ record, json: fact }) => `{"scope":${JSON.stringify(record.entry.scope)},"fact":${fact}}`));
|
|
1158
|
-
const store = options.artifactStore;
|
|
1159
|
-
if (store !== void 0) {
|
|
1160
|
-
const threshold = decode(InlineStringBytesSchema, options.inlineStringBytes ?? DEFAULT_UNIT_INLINE_STRING_BYTES);
|
|
1161
|
-
const entries = writes.map(({ record, json: fact }) => ({
|
|
1162
|
-
scope: record.entry.scope,
|
|
1163
|
-
fact: decodeWith(FactObjectValidator, JSON.parse(fact))
|
|
1164
|
-
}));
|
|
1165
|
-
const encode = async (payload) => {
|
|
1166
|
-
return frame(await Promise.all(entries.map(async ({ scope, fact }) => {
|
|
1167
|
-
const encoded = await (payload ? encodeJournalPayload(fact, store) : encodeJournalFact(fact, store, threshold, options.stringReuse));
|
|
1168
|
-
return JSON.stringify({
|
|
1169
|
-
scope,
|
|
1170
|
-
fact: encoded
|
|
1171
|
-
});
|
|
1172
|
-
})));
|
|
1173
|
-
};
|
|
1174
|
-
json = await encode(utf8ByteLength(json) > JOURNAL_APPEND_BATCH_MAX_BYTES);
|
|
1175
|
-
if (utf8ByteLength(json) > JOURNAL_APPEND_BATCH_MAX_BYTES) json = await encode(true);
|
|
1176
|
-
}
|
|
1177
|
-
if (writes.length <= JOURNAL_APPEND_BATCH_MAX_FACTS && utf8ByteLength(json) <= JOURNAL_APPEND_BATCH_MAX_BYTES) return json;
|
|
1178
|
-
const reference = decode(ReferenceFrameSchema, {
|
|
1179
|
-
format: "sorbet.unit",
|
|
1180
|
-
storageVersion: 2,
|
|
1181
|
-
unitId: first.unitId,
|
|
1182
|
-
epochId: first.epochId,
|
|
1183
|
-
commitId: first.commitId,
|
|
1184
|
-
firstSequence: first.unitSequence,
|
|
1185
|
-
transaction: {
|
|
1186
|
-
id: first.commitId,
|
|
1187
|
-
count: writes.length,
|
|
1188
|
-
byteLength: utf8ByteLength(json)
|
|
1189
|
-
}
|
|
1190
|
-
});
|
|
1191
|
-
const committed = JSON.stringify(reference);
|
|
1192
|
-
if (utf8ByteLength(committed) > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new RangeError("Unit commit identity exceeds the physical frame byte limit");
|
|
1193
|
-
await transactions.write(reference.transaction.id, json);
|
|
1194
|
-
return committed;
|
|
1195
|
-
}
|
|
1196
|
-
/** The unit format is deliberately independent of the single-Session JSONL v4 decoder. */
|
|
1197
|
-
function decodeUnitFrame(input, artifacts, transactions) {
|
|
1198
|
-
try {
|
|
1199
|
-
const frame = readFrame(isString(input) ? input : Utf8Decoder.decode(input), transactions);
|
|
1200
|
-
const records = isPromise(frame) ? frame.then((decoded) => decodeFrameRecords(decoded, artifacts)) : decodeFrameRecords(frame, artifacts);
|
|
1201
|
-
return isPromise(records) ? records.catch(failFrameDecode) : records;
|
|
1202
|
-
} catch (cause) {
|
|
1203
|
-
return failFrameDecode(cause);
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
function decodeFrameRecords(frame, artifacts) {
|
|
1207
|
-
const records = [];
|
|
1208
|
-
const pending = [];
|
|
1209
|
-
for (const [index, entry] of frame.entries.entries()) {
|
|
1210
|
-
const decoded = decodeJournalEnvelope(entry.fact, artifacts);
|
|
1211
|
-
const store = (fact) => {
|
|
1212
|
-
records[index] = {
|
|
1213
|
-
unitId: frame.unitId,
|
|
1214
|
-
epochId: frame.epochId,
|
|
1215
|
-
commitId: frame.commitId,
|
|
1216
|
-
unitSequence: frame.firstSequence + index,
|
|
1217
|
-
entry: entry.scope === "session" ? {
|
|
1218
|
-
scope: entry.scope,
|
|
1219
|
-
fact: decode(CommittedSessionFact, fact)
|
|
1220
|
-
} : {
|
|
1221
|
-
scope: entry.scope,
|
|
1222
|
-
fact: decode(CommittedUnitFact, fact)
|
|
1223
|
-
}
|
|
1224
|
-
};
|
|
1225
|
-
};
|
|
1226
|
-
if (isPromise(decoded)) pending.push(decoded.then(store));
|
|
1227
|
-
else store(decoded);
|
|
1228
|
-
}
|
|
1229
|
-
return pending.length === 0 ? records : Promise.all(pending).then(() => records);
|
|
1230
|
-
}
|
|
1231
|
-
function failFrameDecode(cause) {
|
|
1232
|
-
throw new UnitStoreError("unit_corrupt", "Cannot decode a complete unit frame or its referenced payload", { cause });
|
|
1233
|
-
}
|
|
1234
|
-
|
|
1235
|
-
//#endregion
|
|
1236
|
-
//#region src/journal/unit/scan.ts
|
|
1237
|
-
/**
|
|
1238
|
-
* Reopen reads the whole Journal, so chunks are large: each read is a thread-pool round trip,
|
|
1239
|
-
* and 64 KiB chunks left the open path mostly waiting on them.
|
|
1240
|
-
*/
|
|
1241
|
-
const SCAN_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
1242
|
-
/** Pins the observed file length. The final unterminated bytes are evidence, never a commit. */
|
|
1243
|
-
async function* scanUnitFrames(file, signal) {
|
|
1244
|
-
const { size } = await file.stat();
|
|
1245
|
-
let pending = [];
|
|
1246
|
-
let pendingLength = 0;
|
|
1247
|
-
let position = 0;
|
|
1248
|
-
let offset = 0;
|
|
1249
|
-
while (position < size) {
|
|
1250
|
-
signal?.throwIfAborted();
|
|
1251
|
-
const buffer = Buffer.allocUnsafe(Math.min(SCAN_CHUNK_BYTES, size - position));
|
|
1252
|
-
const { bytesRead } = await file.read(buffer, 0, buffer.length, position);
|
|
1253
|
-
if (bytesRead === 0) throw new UnitStoreError("unit_corrupt", "Unit file was truncated during scan");
|
|
1254
|
-
position += bytesRead;
|
|
1255
|
-
let start = 0;
|
|
1256
|
-
let newline = buffer.indexOf(10, start);
|
|
1257
|
-
while (newline !== -1 && newline < bytesRead) {
|
|
1258
|
-
const length = pendingLength + newline - start;
|
|
1259
|
-
if (length > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new UnitStoreError("unit_corrupt", "Unit frame exceeds its format byte limit");
|
|
1260
|
-
const chunk$1 = buffer.subarray(start, newline);
|
|
1261
|
-
const bytes = pendingLength === 0 ? chunk$1 : Buffer.concat([...pending, chunk$1], length);
|
|
1262
|
-
yield {
|
|
1263
|
-
offset,
|
|
1264
|
-
bytes,
|
|
1265
|
-
complete: true
|
|
1266
|
-
};
|
|
1267
|
-
offset += length + 1;
|
|
1268
|
-
pending = [];
|
|
1269
|
-
pendingLength = 0;
|
|
1270
|
-
start = newline + 1;
|
|
1271
|
-
newline = buffer.indexOf(10, start);
|
|
1272
|
-
}
|
|
1273
|
-
if (start < bytesRead) {
|
|
1274
|
-
pendingLength += bytesRead - start;
|
|
1275
|
-
if (pendingLength > JOURNAL_APPEND_BATCH_MAX_BYTES) throw new UnitStoreError("unit_corrupt", "Unit frame exceeds its format byte limit");
|
|
1276
|
-
pending.push(buffer.subarray(start, bytesRead));
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
if (pendingLength > 0) yield {
|
|
1280
|
-
offset,
|
|
1281
|
-
bytes: Buffer.concat(pending, pendingLength),
|
|
1282
|
-
complete: false
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
//#endregion
|
|
1287
|
-
//#region src/journal/unit/transactions.ts
|
|
1288
|
-
var FileUnitTransactions = class {
|
|
1289
|
-
#directory;
|
|
1290
|
-
constructor(filePath) {
|
|
1291
|
-
this.#directory = `${filePath}.transactions`;
|
|
1292
|
-
}
|
|
1293
|
-
async write(id, text) {
|
|
1294
|
-
await mkdir(this.#directory, {
|
|
1295
|
-
recursive: true,
|
|
1296
|
-
mode: 448
|
|
1297
|
-
});
|
|
1298
|
-
const file = await open(join(this.#directory, `${id}.json`), "wx", 384);
|
|
1299
|
-
try {
|
|
1300
|
-
await file.writeFile(text, "utf8");
|
|
1301
|
-
await file.sync();
|
|
1302
|
-
} finally {
|
|
1303
|
-
await file.close();
|
|
1304
|
-
}
|
|
1305
|
-
await syncDirectory(this.#directory);
|
|
1306
|
-
await syncDirectory(dirname(this.#directory));
|
|
1307
|
-
}
|
|
1308
|
-
async read(id, maxBytes) {
|
|
1309
|
-
const file = await open(join(this.#directory, `${id}.json`), "r");
|
|
1310
|
-
try {
|
|
1311
|
-
if (maxBytes !== void 0 && (await file.stat()).size > maxBytes) throw new UnitStoreError("unit_inspection_capacity", "Transaction exceeds the diagnostic decoding budget");
|
|
1312
|
-
return new TextDecoder("utf-8", { fatal: true }).decode(await file.readFile());
|
|
1313
|
-
} finally {
|
|
1314
|
-
await file.close();
|
|
1315
|
-
}
|
|
1316
|
-
}
|
|
1317
|
-
};
|
|
1318
|
-
|
|
1319
|
-
//#endregion
|
|
1320
|
-
//#region src/journal/unit/seal.ts
|
|
1321
|
-
const compress = promisify(brotliCompress);
|
|
1322
|
-
const decompress = promisify(brotliDecompress);
|
|
1323
|
-
const SealedUnitManifestSchema = s.obj({
|
|
1324
|
-
format: s.lit("sorbet.unit.sealed"),
|
|
1325
|
-
storageVersion: s.lit(1),
|
|
1326
|
-
codec: s.lit("brotli"),
|
|
1327
|
-
journalBytes: s.int({ minimum: 1 }),
|
|
1328
|
-
sealedBytes: s.int({ minimum: 1 }),
|
|
1329
|
-
sealedAt: s.str({ minLength: 1 })
|
|
1330
|
-
}, { open: false });
|
|
1331
|
-
function sealedUnitLayout(journalPath) {
|
|
1332
|
-
return {
|
|
1333
|
-
journalPath,
|
|
1334
|
-
sealedPath: `${journalPath}.sealed`,
|
|
1335
|
-
manifestPath: `${journalPath}.sealed.json`
|
|
1336
|
-
};
|
|
1337
|
-
}
|
|
1338
|
-
async function readSealedUnitManifest(journalPath) {
|
|
1339
|
-
const { manifestPath } = sealedUnitLayout(journalPath);
|
|
1340
|
-
const text = await readFile(manifestPath, "utf8").catch(() => void 0);
|
|
1341
|
-
if (text === void 0) return void 0;
|
|
1342
|
-
try {
|
|
1343
|
-
return decode(SealedUnitManifestSchema, JSON.parse(text));
|
|
1344
|
-
} catch (cause) {
|
|
1345
|
-
throw new UnitStoreError("unit_corrupt", "Sealed Unit manifest is invalid", { cause });
|
|
1346
|
-
}
|
|
1347
|
-
}
|
|
1348
|
-
async function isUnitSealed(journalPath) {
|
|
1349
|
-
if (await readSealedUnitManifest(journalPath) === void 0) return false;
|
|
1350
|
-
return await exists(journalPath) === false;
|
|
1351
|
-
}
|
|
1352
|
-
/**
|
|
1353
|
-
* Reads the committed bytes of a sealed Unit without restoring the plain Journal. Readers use
|
|
1354
|
-
* this so an archived Unit stays compressed until something actually needs to append to it.
|
|
1355
|
-
*/
|
|
1356
|
-
async function readSealedUnitJournal(journalPath) {
|
|
1357
|
-
const manifest = await readSealedUnitManifest(journalPath);
|
|
1358
|
-
if (manifest === void 0) return void 0;
|
|
1359
|
-
if (await exists(journalPath)) return void 0;
|
|
1360
|
-
const { sealedPath } = sealedUnitLayout(journalPath);
|
|
1361
|
-
const sealed = await readFile(sealedPath).catch(() => void 0);
|
|
1362
|
-
if (sealed === void 0) throw new UnitStoreError("unit_corrupt", "Sealed Unit payload is missing");
|
|
1363
|
-
const bytes = await decompress(sealed);
|
|
1364
|
-
if (bytes.byteLength !== manifest.journalBytes) throw new UnitStoreError("unit_corrupt", "Sealed Unit payload does not match its manifest");
|
|
1365
|
-
return {
|
|
1366
|
-
bytes,
|
|
1367
|
-
manifest
|
|
1368
|
-
};
|
|
1369
|
-
}
|
|
1370
|
-
/** Compresses a closed Unit's committed Journal and verifies the result before replacing it. */
|
|
1371
|
-
async function sealUnit(journalPath, options = {}) {
|
|
1372
|
-
const { sealedPath, manifestPath } = sealedUnitLayout(journalPath);
|
|
1373
|
-
const lease = await acquireJournalWriterLease(journalPath).catch((cause) => {
|
|
1374
|
-
if (cause instanceof JournalWriterBusyError) throw new UnitStoreError("unit_locked", "A sealed Unit must not have a live writer", { cause });
|
|
1375
|
-
throw cause;
|
|
1376
|
-
});
|
|
1377
|
-
try {
|
|
1378
|
-
return await sealWithLease(journalPath, sealedPath, manifestPath, options);
|
|
1379
|
-
} finally {
|
|
1380
|
-
lease.release();
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
async function sealWithLease(journalPath, sealedPath, manifestPath, options) {
|
|
1384
|
-
if (await exists(journalPath) === false) {
|
|
1385
|
-
const existing = await readSealedUnitManifest(journalPath);
|
|
1386
|
-
if (existing === void 0) throw new UnitStoreError("unit_corrupt", "Unit Journal is missing and no seal records it");
|
|
1387
|
-
return {
|
|
1388
|
-
manifest: existing,
|
|
1389
|
-
journalBytes: existing.journalBytes,
|
|
1390
|
-
sealedBytes: existing.sealedBytes
|
|
1391
|
-
};
|
|
1392
|
-
}
|
|
1393
|
-
const journal = await readFile(journalPath);
|
|
1394
|
-
if (journal.byteLength === 0) throw new UnitStoreError("unit_corrupt", "An empty Unit Journal cannot be sealed");
|
|
1395
|
-
const sealed = await compress(journal, { params: {
|
|
1396
|
-
[constants.BROTLI_PARAM_QUALITY]: 5,
|
|
1397
|
-
[constants.BROTLI_PARAM_SIZE_HINT]: journal.byteLength
|
|
1398
|
-
} });
|
|
1399
|
-
await writeAtomically(sealedPath, sealed);
|
|
1400
|
-
if ((await decompress(await readFile(sealedPath))).equals(journal) === false) {
|
|
1401
|
-
await rm(sealedPath, { force: true });
|
|
1402
|
-
throw new UnitStoreError("unit_corrupt", "Sealed Unit does not reproduce the Journal");
|
|
1403
|
-
}
|
|
1404
|
-
const manifest = decode(SealedUnitManifestSchema, {
|
|
1405
|
-
format: "sorbet.unit.sealed",
|
|
1406
|
-
storageVersion: 1,
|
|
1407
|
-
codec: "brotli",
|
|
1408
|
-
journalBytes: journal.byteLength,
|
|
1409
|
-
sealedBytes: sealed.byteLength,
|
|
1410
|
-
sealedAt: (options.now?.() ?? /* @__PURE__ */ new Date()).toISOString()
|
|
1411
|
-
});
|
|
1412
|
-
await writeAtomically(manifestPath, Buffer.from(`${JSON.stringify(manifest)}\n`, "utf8"));
|
|
1413
|
-
await rm(journalPath);
|
|
1414
|
-
await syncDirectory(dirname(journalPath));
|
|
1415
|
-
return {
|
|
1416
|
-
manifest,
|
|
1417
|
-
journalBytes: journal.byteLength,
|
|
1418
|
-
sealedBytes: sealed.byteLength
|
|
1419
|
-
};
|
|
1420
|
-
}
|
|
1421
|
-
/** Restores a sealed Unit to its exact committed bytes. A present Journal wins over any seal. */
|
|
1422
|
-
async function unsealUnit(journalPath, options = {}) {
|
|
1423
|
-
const { sealedPath, manifestPath } = sealedUnitLayout(journalPath);
|
|
1424
|
-
const manifest = await readSealedUnitManifest(journalPath);
|
|
1425
|
-
if (await exists(journalPath)) if (options.discardJournal === true) await rm(journalPath);
|
|
1426
|
-
else {
|
|
1427
|
-
if (manifest !== void 0) await discardSeal(sealedPath, manifestPath);
|
|
1428
|
-
return { restored: false };
|
|
1429
|
-
}
|
|
1430
|
-
if (manifest === void 0) {
|
|
1431
|
-
if (await exists(journalPath) === false) return { restored: false };
|
|
1432
|
-
throw new UnitStoreError("unit_corrupt", "Sealed Unit Journal is missing and unrecorded");
|
|
1433
|
-
}
|
|
1434
|
-
const sealed = await readSealedUnitJournal(journalPath).catch(async (cause) => {
|
|
1435
|
-
await rm(sealedPath, { force: true });
|
|
1436
|
-
await rm(manifestPath, { force: true });
|
|
1437
|
-
throw cause;
|
|
1438
|
-
});
|
|
1439
|
-
if (sealed === void 0) throw new UnitStoreError("unit_corrupt", "Sealed Unit payload is missing");
|
|
1440
|
-
await writeAtomically(journalPath, sealed.bytes);
|
|
1441
|
-
await discardSeal(sealedPath, manifestPath);
|
|
1442
|
-
return { restored: true };
|
|
1443
|
-
}
|
|
1444
|
-
/** Removes a seal that cannot be trusted because the Journal it describes is still present. */
|
|
1445
|
-
async function discardIncompleteSeal(journalPath) {
|
|
1446
|
-
const { sealedPath, manifestPath } = sealedUnitLayout(journalPath);
|
|
1447
|
-
if (await exists(journalPath)) await discardSeal(sealedPath, manifestPath);
|
|
1448
|
-
}
|
|
1449
|
-
async function discardSeal(sealedPath, manifestPath) {
|
|
1450
|
-
await rm(sealedPath, { force: true });
|
|
1451
|
-
await rm(manifestPath, { force: true });
|
|
1452
|
-
await syncDirectory(dirname(manifestPath));
|
|
1453
|
-
}
|
|
1454
|
-
async function writeAtomically(path, bytes) {
|
|
1455
|
-
const staging = `${path}.tmp`;
|
|
1456
|
-
const handle = await open(staging, "w", 384);
|
|
1457
|
-
try {
|
|
1458
|
-
await handle.writeFile(bytes);
|
|
1459
|
-
await handle.sync();
|
|
1460
|
-
} finally {
|
|
1461
|
-
await handle.close();
|
|
1462
|
-
}
|
|
1463
|
-
await rename(staging, path);
|
|
1464
|
-
await syncDirectory(dirname(path));
|
|
1465
|
-
}
|
|
1466
|
-
async function exists(path) {
|
|
1467
|
-
return await stat(path).then(() => true, () => false);
|
|
1468
|
-
}
|
|
1469
|
-
|
|
1470
|
-
//#endregion
|
|
1471
|
-
//#region src/journal/unit/jsonl.ts
|
|
1472
|
-
function sealedScanSource(bytes) {
|
|
1473
|
-
return {
|
|
1474
|
-
stat: async () => ({ size: bytes.byteLength }),
|
|
1475
|
-
read: async (buffer, offset, length, position) => {
|
|
1476
|
-
const end = Math.min(position + length, bytes.byteLength);
|
|
1477
|
-
if (end <= position) return { bytesRead: 0 };
|
|
1478
|
-
bytes.copy(buffer, offset, position, end);
|
|
1479
|
-
return { bytesRead: end - position };
|
|
1480
|
-
},
|
|
1481
|
-
close: async () => {}
|
|
1482
|
-
};
|
|
1483
|
-
}
|
|
1484
|
-
async function openUnitScanSource(path, mode) {
|
|
1485
|
-
const sealed = await readSealedUnitJournal(path);
|
|
1486
|
-
if (sealed !== void 0) return {
|
|
1487
|
-
source: sealedScanSource(sealed.bytes),
|
|
1488
|
-
sealed: true
|
|
1489
|
-
};
|
|
1490
|
-
return {
|
|
1491
|
-
source: await open(path, mode),
|
|
1492
|
-
sealed: false
|
|
1493
|
-
};
|
|
1494
|
-
}
|
|
1495
|
-
/**
|
|
1496
|
-
* Sealing is an offline archive step: a sealed Unit has no readable Journal until it is
|
|
1497
|
-
* unsealed, so opening one must fail loudly instead of looking like an empty Session.
|
|
1498
|
-
*/
|
|
1499
|
-
async function assertNotSealed(filePath) {
|
|
1500
|
-
if (await isUnitSealed(filePath)) throw new UnitStoreError("unit_sealed", "Unit Journal is sealed; unseal it before opening the Unit");
|
|
1501
|
-
await discardIncompleteSeal(filePath);
|
|
1502
|
-
}
|
|
1503
|
-
/**
|
|
1504
|
-
* Bounded reuse for the several Journal passes made while opening one Unit
|
|
1505
|
-
* (history, Session participants, command coverage, replay labels). Above the
|
|
1506
|
-
* bound the raw records would cost more memory than the repeated decode saves.
|
|
1507
|
-
*/
|
|
1508
|
-
const DECODED_FRAME_CACHE_MAX_BYTES = 16 * 1024 * 1024;
|
|
1509
|
-
/** Repeat-read cache budget in raw frame bytes; the decoded records are several times larger. */
|
|
1510
|
-
const DECODED_FRAME_READ_CACHE_BYTES = 4 * 1024 * 1024;
|
|
1511
|
-
function sharedPrefix(values) {
|
|
1512
|
-
const [first = "", ...rest] = values;
|
|
1513
|
-
let length = first.length;
|
|
1514
|
-
for (const value of rest) {
|
|
1515
|
-
let index = 0;
|
|
1516
|
-
while (index < length && value[index] === first[index]) index++;
|
|
1517
|
-
length = index;
|
|
1518
|
-
}
|
|
1519
|
-
return first.slice(0, length);
|
|
1520
|
-
}
|
|
1521
|
-
/** Version 2 frames are markers whose facts live in a transaction side file. */
|
|
1522
|
-
const ReferenceFrameMarker = Buffer.from("\"storageVersion\":2");
|
|
1523
|
-
var UnitFrames = class {
|
|
1524
|
-
#positions = [];
|
|
1525
|
-
#frames = [];
|
|
1526
|
-
#decoded = [];
|
|
1527
|
-
#cacheOrder = [];
|
|
1528
|
-
#cacheFrames = false;
|
|
1529
|
-
#cacheBytes = 0;
|
|
1530
|
-
completeBytes = 0;
|
|
1531
|
-
incompleteTail = false;
|
|
1532
|
-
constructor(file, transactions, artifacts, cacheDecoded = false) {
|
|
1533
|
-
this.file = file;
|
|
1534
|
-
this.transactions = transactions;
|
|
1535
|
-
this.artifacts = artifacts;
|
|
1536
|
-
this.cacheDecoded = cacheDecoded;
|
|
1537
|
-
}
|
|
1538
|
-
async scan(visit) {
|
|
1539
|
-
this.#positions.length = 0;
|
|
1540
|
-
this.#frames.length = 0;
|
|
1541
|
-
this.#decoded.length = 0;
|
|
1542
|
-
this.#cacheOrder.length = 0;
|
|
1543
|
-
this.#cacheBytes = 0;
|
|
1544
|
-
this.completeBytes = 0;
|
|
1545
|
-
this.incompleteTail = false;
|
|
1546
|
-
const cache = this.cacheDecoded && (await this.file.stat()).size <= DECODED_FRAME_CACHE_MAX_BYTES;
|
|
1547
|
-
this.#cacheFrames = cache;
|
|
1548
|
-
const inFlight = [];
|
|
1549
|
-
const settle = async () => {
|
|
1550
|
-
const next = inFlight.shift();
|
|
1551
|
-
if (next === void 0) return;
|
|
1552
|
-
const records = isPromise(next.records) ? await next.records : next.records;
|
|
1553
|
-
visit(records);
|
|
1554
|
-
if (cache) this.#decoded[this.#frames.length] = records;
|
|
1555
|
-
this.add({
|
|
1556
|
-
offset: next.offset,
|
|
1557
|
-
length: next.length
|
|
1558
|
-
}, records.length);
|
|
1559
|
-
this.completeBytes = next.offset + next.length + 1;
|
|
1560
|
-
};
|
|
1561
|
-
try {
|
|
1562
|
-
for await (const frame of scanUnitFrames(this.file)) {
|
|
1563
|
-
if (!frame.complete) {
|
|
1564
|
-
this.incompleteTail = true;
|
|
1565
|
-
break;
|
|
1566
|
-
}
|
|
1567
|
-
const records = decodeUnitFrame(frame.bytes, this.artifacts, this.transactions);
|
|
1568
|
-
if (isPromise(records)) records.catch(() => {});
|
|
1569
|
-
inFlight.push({
|
|
1570
|
-
offset: frame.offset,
|
|
1571
|
-
length: frame.bytes.length,
|
|
1572
|
-
records
|
|
1573
|
-
});
|
|
1574
|
-
while (inFlight.length > 0 && (inFlight.length >= 64 || !isPromise(inFlight[0]?.records))) await settle();
|
|
1575
|
-
}
|
|
1576
|
-
while (inFlight.length > 0) await settle();
|
|
1577
|
-
} finally {
|
|
1578
|
-
inFlight.length = 0;
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
add(position, count) {
|
|
1582
|
-
const frame = this.#frames.length;
|
|
1583
|
-
this.#frames.push(position);
|
|
1584
|
-
for (let index = 0; index < count; index++) this.#positions.push({
|
|
1585
|
-
frame,
|
|
1586
|
-
index
|
|
1587
|
-
});
|
|
1588
|
-
}
|
|
1589
|
-
/**
|
|
1590
|
-
* Readers that refresh after every append asked for the same frames repeatedly. Keep the
|
|
1591
|
-
* most recent decodes so an unchanged frame costs nothing on the next read, bounded by the
|
|
1592
|
-
* raw bytes it came from: decoded records are several times larger than their frame.
|
|
1593
|
-
*/
|
|
1594
|
-
#remember(frameIndex, records) {
|
|
1595
|
-
if (!this.#cacheFrames || this.#decoded[frameIndex] !== void 0) return;
|
|
1596
|
-
const frame = this.#frames[frameIndex];
|
|
1597
|
-
if (frame === void 0) return;
|
|
1598
|
-
this.#decoded[frameIndex] = records;
|
|
1599
|
-
this.#cacheOrder.push(frameIndex);
|
|
1600
|
-
this.#cacheBytes += frame.length;
|
|
1601
|
-
while (this.#cacheBytes > DECODED_FRAME_READ_CACHE_BYTES) {
|
|
1602
|
-
const evicted = this.#cacheOrder.shift();
|
|
1603
|
-
if (evicted === void 0) break;
|
|
1604
|
-
this.#cacheBytes -= this.#frames[evicted]?.length ?? 0;
|
|
1605
|
-
this.#decoded[evicted] = void 0;
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
async read(positions) {
|
|
1609
|
-
const decoded = /* @__PURE__ */ new Map();
|
|
1610
|
-
const needed = /* @__PURE__ */ new Set();
|
|
1611
|
-
for (const position of positions) {
|
|
1612
|
-
const locator = this.#positions[position];
|
|
1613
|
-
if (locator === void 0) throw new UnitStoreError("unit_corrupt", "Missing unit record locator");
|
|
1614
|
-
const cached = this.#decoded[locator.frame];
|
|
1615
|
-
if (cached === void 0) needed.add(locator.frame);
|
|
1616
|
-
else decoded.set(locator.frame, cached);
|
|
1617
|
-
}
|
|
1618
|
-
const ranges = [];
|
|
1619
|
-
for (const index of [...needed].sort((a, b) => a - b)) {
|
|
1620
|
-
const frame = this.#frames[index];
|
|
1621
|
-
if (frame === void 0) throw new UnitStoreError("unit_corrupt", "Missing unit frame locator");
|
|
1622
|
-
const previous = ranges.at(-1);
|
|
1623
|
-
const end = frame.offset + frame.length;
|
|
1624
|
-
if (previous !== void 0 && end - previous.offset <= 64 * 1024) {
|
|
1625
|
-
previous.end = end;
|
|
1626
|
-
previous.frames.push(index);
|
|
1627
|
-
} else ranges.push({
|
|
1628
|
-
offset: frame.offset,
|
|
1629
|
-
end,
|
|
1630
|
-
frames: [index]
|
|
1631
|
-
});
|
|
1632
|
-
}
|
|
1633
|
-
for (let index = 0; index < ranges.length; index += 2) {
|
|
1634
|
-
const loaded = await Promise.allSettled(ranges.slice(index, index + 2).map(async (range) => {
|
|
1635
|
-
const buffer = Buffer.alloc(range.end - range.offset);
|
|
1636
|
-
let read = 0;
|
|
1637
|
-
while (read < buffer.length) {
|
|
1638
|
-
const { bytesRead } = await this.file.read(buffer, read, buffer.length - read, range.offset + read);
|
|
1639
|
-
if (bytesRead === 0) throw new UnitStoreError("unit_corrupt", "Committed unit frame disappeared");
|
|
1640
|
-
read += bytesRead;
|
|
1641
|
-
}
|
|
1642
|
-
await Promise.all(range.frames.map(async (frameIndex) => {
|
|
1643
|
-
const frame = this.#frames[frameIndex];
|
|
1644
|
-
if (frame === void 0) throw new UnitStoreError("unit_corrupt", "Missing unit frame locator");
|
|
1645
|
-
const start = frame.offset - range.offset;
|
|
1646
|
-
const records = decodeUnitFrame(buffer.subarray(start, start + frame.length), this.artifacts, this.transactions);
|
|
1647
|
-
const settled = isPromise(records) ? await records : records;
|
|
1648
|
-
decoded.set(frameIndex, settled);
|
|
1649
|
-
this.#remember(frameIndex, settled);
|
|
1650
|
-
}));
|
|
1651
|
-
}));
|
|
1652
|
-
for (const result of loaded) if (result.status === "rejected") throw result.reason;
|
|
1653
|
-
}
|
|
1654
|
-
return positions.map((position) => {
|
|
1655
|
-
const locator = this.#positions[position];
|
|
1656
|
-
const record = locator === void 0 ? void 0 : decoded.get(locator.frame)?.[locator.index];
|
|
1657
|
-
if (record === void 0) throw new UnitStoreError("unit_corrupt", "Committed unit record disappeared");
|
|
1658
|
-
return record;
|
|
1659
|
-
});
|
|
1660
|
-
}
|
|
1661
|
-
};
|
|
1662
|
-
var JsonlUnitStorage = class {
|
|
1663
|
-
durability = "sync";
|
|
1664
|
-
#frames;
|
|
1665
|
-
#transactions;
|
|
1666
|
-
#stringReuse = new ExternalizedStringReuse();
|
|
1667
|
-
constructor(options, file, lease) {
|
|
1668
|
-
this.options = options;
|
|
1669
|
-
this.file = file;
|
|
1670
|
-
this.lease = lease;
|
|
1671
|
-
this.#transactions = new FileUnitTransactions(options.filePath);
|
|
1672
|
-
this.#frames = new UnitFrames(file, this.#transactions, options.artifactStore, true);
|
|
1673
|
-
}
|
|
1674
|
-
get unitId() {
|
|
1675
|
-
return this.options.unitId;
|
|
1676
|
-
}
|
|
1677
|
-
async load(visit) {
|
|
1678
|
-
await this.#frames.scan(visit);
|
|
1679
|
-
if ((await this.file.stat()).size !== this.#frames.completeBytes) {
|
|
1680
|
-
await this.file.truncate(this.#frames.completeBytes);
|
|
1681
|
-
await this.file.sync();
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
read(positions) {
|
|
1685
|
-
return this.#frames.read(positions);
|
|
1686
|
-
}
|
|
1687
|
-
async append(writes) {
|
|
1688
|
-
let json;
|
|
1689
|
-
try {
|
|
1690
|
-
json = await encodeUnitFrame(writes, {
|
|
1691
|
-
...this.options,
|
|
1692
|
-
stringReuse: this.#stringReuse
|
|
1693
|
-
}, this.#transactions);
|
|
1694
|
-
} catch (cause) {
|
|
1695
|
-
return {
|
|
1696
|
-
status: "not-committed",
|
|
1697
|
-
error: ensureError(cause)
|
|
1698
|
-
};
|
|
1699
|
-
}
|
|
1700
|
-
const offset = this.#frames.completeBytes;
|
|
1701
|
-
const bytes = Buffer.from(`${json}\n`);
|
|
1702
|
-
try {
|
|
1703
|
-
let written = 0;
|
|
1704
|
-
while (written < bytes.length) {
|
|
1705
|
-
const { bytesWritten } = await this.file.write(bytes, written, bytes.length - written, offset + written);
|
|
1706
|
-
if (bytesWritten === 0) throw new Error("Unit file write made no progress");
|
|
1707
|
-
written += bytesWritten;
|
|
1708
|
-
}
|
|
1709
|
-
await this.file.sync();
|
|
1710
|
-
this.#frames.add({
|
|
1711
|
-
offset,
|
|
1712
|
-
length: bytes.length - 1
|
|
1713
|
-
}, writes.length);
|
|
1714
|
-
this.#frames.completeBytes += bytes.length;
|
|
1715
|
-
return { status: "committed" };
|
|
1716
|
-
} catch (cause) {
|
|
1717
|
-
return {
|
|
1718
|
-
status: "unknown",
|
|
1719
|
-
error: ensureError(cause)
|
|
1720
|
-
};
|
|
1721
|
-
}
|
|
1722
|
-
}
|
|
1723
|
-
async close() {
|
|
1724
|
-
try {
|
|
1725
|
-
await this.file.close();
|
|
1726
|
-
} finally {
|
|
1727
|
-
this.lease.release();
|
|
1728
|
-
}
|
|
1729
|
-
}
|
|
1730
|
-
};
|
|
1731
|
-
var JsonlUnitReader = class {
|
|
1732
|
-
#closing;
|
|
1733
|
-
#queue = Promise.resolve();
|
|
1734
|
-
constructor(file, frames, history, incompleteTail) {
|
|
1735
|
-
this.file = file;
|
|
1736
|
-
this.frames = frames;
|
|
1737
|
-
this.history = history;
|
|
1738
|
-
this.incompleteTail = incompleteTail;
|
|
1739
|
-
}
|
|
1740
|
-
get unitId() {
|
|
1741
|
-
return this.history.unitId;
|
|
1742
|
-
}
|
|
1743
|
-
get tail() {
|
|
1744
|
-
return this.history.tail;
|
|
1745
|
-
}
|
|
1746
|
-
read(options = {}) {
|
|
1747
|
-
if (this.#closing !== void 0) return Promise.reject(new UnitStoreError("unit_closed", "Unit reader is closed"));
|
|
1748
|
-
const { positions, hasMore } = this.history.positions(options);
|
|
1749
|
-
const work = this.#queue.then(async () => {
|
|
1750
|
-
const records = await this.frames.read(positions);
|
|
1751
|
-
const last = records.at(-1);
|
|
1752
|
-
return {
|
|
1753
|
-
records,
|
|
1754
|
-
cursor: last === void 0 ? options.after : unitRecordCursor(last),
|
|
1755
|
-
hasMore
|
|
1756
|
-
};
|
|
1757
|
-
});
|
|
1758
|
-
this.#queue = work.then(() => {}, () => {});
|
|
1759
|
-
return work;
|
|
1760
|
-
}
|
|
1761
|
-
close() {
|
|
1762
|
-
this.#closing ??= this.#queue.then(() => this.file.close());
|
|
1763
|
-
return this.#closing;
|
|
1764
|
-
}
|
|
1765
|
-
};
|
|
1766
|
-
/** One tree per file. Fact bodies stay on disk; the opened store indexes frame positions. */
|
|
1767
|
-
var JsonlUnitStore = class {
|
|
1768
|
-
#options;
|
|
1769
|
-
constructor(options) {
|
|
1770
|
-
this.#options = { ...options };
|
|
1771
|
-
}
|
|
1772
|
-
/** Diagnostic scan with a transaction decoding budget; never claims a writer. */
|
|
1773
|
-
inspect(options = {}) {
|
|
1774
|
-
return this.#readLifecycle(options, true);
|
|
1775
|
-
}
|
|
1776
|
-
/** Complete lifecycle scan without diagnostic budgets or historical content hydration. */
|
|
1777
|
-
readRecovery(options) {
|
|
1778
|
-
return this.#readLifecycle(options, false);
|
|
1779
|
-
}
|
|
1780
|
-
async *#readLifecycle(options, diagnostic) {
|
|
1781
|
-
const unitKinds = diagnostic ? options.unitKinds : [];
|
|
1782
|
-
const needle = options.sessionId === void 0 && unitKinds !== void 0 ? Buffer.from(sharedPrefix(unitKinds.map((kind) => `"kind":${JSON.stringify(kind)}`))) : void 0;
|
|
1783
|
-
const path = await canonicalJournalPath(this.#options.filePath);
|
|
1784
|
-
const { source: file } = await openUnitScanSource(path, "r");
|
|
1785
|
-
const transactions = new FileUnitTransactions(path);
|
|
1786
|
-
let sequence = 1;
|
|
1787
|
-
let sessionSequence = 1;
|
|
1788
|
-
let frames = 0;
|
|
1789
|
-
let skipped = false;
|
|
1790
|
-
try {
|
|
1791
|
-
for await (const raw of scanUnitFrames(file, options.signal)) {
|
|
1792
|
-
if (!raw.complete) {
|
|
1793
|
-
yield {
|
|
1794
|
-
unitFacts: [],
|
|
1795
|
-
sessionFacts: [],
|
|
1796
|
-
incompleteTail: true
|
|
1797
|
-
};
|
|
1798
|
-
return;
|
|
1799
|
-
}
|
|
1800
|
-
frames++;
|
|
1801
|
-
if (needle !== void 0 && !raw.bytes.subarray(0, 64).includes(ReferenceFrameMarker) && !raw.bytes.includes(needle)) {
|
|
1802
|
-
skipped = true;
|
|
1803
|
-
continue;
|
|
1804
|
-
}
|
|
1805
|
-
const frame = await decodeUnitInspectionFrame(raw.bytes, this.#options.artifactStore, transactions, options.sessionId, diagnostic, unitKinds === void 0 ? void 0 : new Set(unitKinds));
|
|
1806
|
-
if (frame.unitId !== this.#options.unitId || (skipped ? frame.firstSequence < sequence : frame.firstSequence !== sequence)) throw new UnitStoreError("unit_corrupt", "Unit inspection identity or ordering differs");
|
|
1807
|
-
skipped = false;
|
|
1808
|
-
sequence = frame.firstSequence + frame.count;
|
|
1809
|
-
for (const fact of frame.sessionFacts) if (fact.sequence !== sessionSequence++ || fact.cursor !== fact.id) throw new UnitStoreError("unit_corrupt", "Session inspection ordering differs");
|
|
1810
|
-
yield {
|
|
1811
|
-
unitFacts: frame.unitFacts,
|
|
1812
|
-
sessionFacts: frame.sessionFacts,
|
|
1813
|
-
incompleteTail: false
|
|
1814
|
-
};
|
|
1815
|
-
}
|
|
1816
|
-
if (frames === 0) throw new UnitStoreError("unit_corrupt", "Published unit has no committed catalog");
|
|
1817
|
-
} finally {
|
|
1818
|
-
await file.close();
|
|
1819
|
-
}
|
|
1820
|
-
}
|
|
1821
|
-
async open(options = {}) {
|
|
1822
|
-
await assertNotSealed(this.#options.filePath);
|
|
1823
|
-
const directory = dirname(resolve(this.#options.filePath));
|
|
1824
|
-
const created = await mkdir(directory, {
|
|
1825
|
-
recursive: true,
|
|
1826
|
-
mode: 448
|
|
1827
|
-
});
|
|
1828
|
-
if (created !== void 0) {
|
|
1829
|
-
let current = directory;
|
|
1830
|
-
while (current !== dirname(created)) {
|
|
1831
|
-
await syncDirectory(current);
|
|
1832
|
-
current = dirname(current);
|
|
1833
|
-
}
|
|
1834
|
-
await syncDirectory(dirname(created));
|
|
1835
|
-
}
|
|
1836
|
-
const path = await canonicalJournalPath(this.#options.filePath);
|
|
1837
|
-
let lease;
|
|
1838
|
-
try {
|
|
1839
|
-
lease = await acquireJournalWriterLease(path);
|
|
1840
|
-
} catch (cause) {
|
|
1841
|
-
if (cause instanceof JournalWriterBusyError) throw new UnitStoreError("unit_locked", `Unit ${this.#options.unitId} already has a writer`, { cause });
|
|
1842
|
-
throw cause;
|
|
1843
|
-
}
|
|
1844
|
-
try {
|
|
1845
|
-
await createJournalFile(path);
|
|
1846
|
-
const file = await open(path, "r+");
|
|
1847
|
-
return await openUnitStorage(new JsonlUnitStorage({
|
|
1848
|
-
...this.#options,
|
|
1849
|
-
filePath: path
|
|
1850
|
-
}, file, lease), options);
|
|
1851
|
-
} catch (cause) {
|
|
1852
|
-
lease.release();
|
|
1853
|
-
throw cause;
|
|
1854
|
-
}
|
|
1855
|
-
}
|
|
1856
|
-
/**
|
|
1857
|
-
* Pins a complete observed prefix without writer ownership or tail repair.
|
|
1858
|
-
* `sync` persists that prefix before returning, for forks that must outlive source loss.
|
|
1859
|
-
*/
|
|
1860
|
-
async openReader(options = {}) {
|
|
1861
|
-
const path = await canonicalJournalPath(this.#options.filePath);
|
|
1862
|
-
const { source: file, sealed } = await openUnitScanSource(path, options.durability === "sync" ? "r+" : "r");
|
|
1863
|
-
try {
|
|
1864
|
-
const frames = new UnitFrames(file, new FileUnitTransactions(path), this.#options.artifactStore);
|
|
1865
|
-
const history = new UnitHistory(this.#options.unitId);
|
|
1866
|
-
await frames.scan((records) => assertUnitHistory(history, records));
|
|
1867
|
-
if (options.durability === "sync" && !sealed) await file.sync?.();
|
|
1868
|
-
return new JsonlUnitReader(file, frames, history, frames.incompleteTail);
|
|
1869
|
-
} catch (cause) {
|
|
1870
|
-
await file.close();
|
|
1871
|
-
throw cause;
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
};
|
|
1875
|
-
|
|
1876
|
-
//#endregion
|
|
1877
|
-
export { JsonlJournalReader, JsonlJournalStore, JsonlUnitStore, acquireJournalWriterLease, discardIncompleteSeal, isUnitSealed, readJsonlJournal, readSealedUnitManifest, sealUnit, sealedUnitLayout, unsealUnit };
|
|
1878
|
-
//# sourceMappingURL=node.js.map
|
|
3
|
+
export { JsonlJournalReader, JsonlJournalStore, JsonlUnitStore, acquireJournalWriterLease, discardIncompleteSeal, isUnitSealed, readJsonlJournal, readSealedUnitManifest, readUnitSessions, sealUnit, sealedUnitLayout, unsealUnit };
|