@gajae-code/coding-agent 0.12.19 → 0.12.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +20 -9
  2. package/dist/types/commands/gc.d.ts +4 -0
  3. package/dist/types/commands/sdk.d.ts +66 -0
  4. package/dist/types/config/settings-schema.d.ts +34 -0
  5. package/dist/types/gjc-runtime/gc-runtime.d.ts +97 -1
  6. package/dist/types/modes/acp/acp-agent.d.ts +33 -1
  7. package/dist/types/modes/components/bash-execution.d.ts +7 -0
  8. package/dist/types/modes/components/eval-execution.d.ts +7 -0
  9. package/dist/types/modes/utils/ui-helpers.d.ts +10 -1
  10. package/dist/types/sdk/acp/adapter.d.ts +9 -17
  11. package/dist/types/sdk/acp/index.d.ts +1 -1
  12. package/dist/types/sdk/broker/broker.d.ts +93 -1
  13. package/dist/types/sdk/broker/ensure.d.ts +17 -0
  14. package/dist/types/sdk/broker/index.d.ts +1 -0
  15. package/dist/types/sdk/broker/lifecycle-ledger.d.ts +10 -1
  16. package/dist/types/sdk/broker/lifecycle.d.ts +81 -2
  17. package/dist/types/sdk/broker/startup-budget.d.ts +44 -0
  18. package/dist/types/sdk/bus/chat-daemon-control.d.ts +7 -1
  19. package/dist/types/sdk/bus/chat-daemon-runtime.d.ts +11 -1
  20. package/dist/types/sdk/bus/discord-daemon.d.ts +1 -0
  21. package/dist/types/sdk/bus/index.d.ts +20 -9
  22. package/dist/types/sdk/bus/prompt-reconciliation.d.ts +2 -6
  23. package/dist/types/sdk/bus/slack-daemon.d.ts +2 -2
  24. package/dist/types/sdk/host/query/revision-store.d.ts +5 -0
  25. package/dist/types/sdk/host/session-runtime.d.ts +4 -0
  26. package/dist/types/sdk/lifecycle-session.d.ts +13 -2
  27. package/dist/types/sdk/prompt-failure.d.ts +13 -0
  28. package/dist/types/sdk/prompt-watchdog.d.ts +82 -0
  29. package/dist/types/sdk/session-reconnect.d.ts +20 -0
  30. package/dist/types/sdk/startup-capability.d.ts +1 -1
  31. package/dist/types/session/agent-session.d.ts +14 -4
  32. package/dist/types/session/blob-store.d.ts +32 -0
  33. package/dist/types/session/internal/managed-session-storage.d.ts +40 -2
  34. package/dist/types/session/internal/session-memory-sidecar.d.ts +881 -0
  35. package/dist/types/session/session-manager.d.ts +301 -27
  36. package/dist/types/session/session-storage.d.ts +170 -0
  37. package/dist/types/task/executor.d.ts +2 -1
  38. package/dist/types/tools/bash.d.ts +1 -0
  39. package/dist/types/tui/output-block.d.ts +6 -0
  40. package/package.json +8 -8
  41. package/src/cli/args.ts +75 -34
  42. package/src/commands/acp.ts +7 -1
  43. package/src/commands/gc.ts +31 -2
  44. package/src/commands/sdk.ts +141 -7
  45. package/src/config/settings-schema.ts +43 -0
  46. package/src/coordinator-mcp/server.ts +6 -1
  47. package/src/export/html/index.ts +83 -39
  48. package/src/gjc-runtime/gc-runtime.ts +1283 -4
  49. package/src/internal-urls/docs-index.generated.ts +1 -1
  50. package/src/internal-urls/local-protocol.ts +14 -0
  51. package/src/main.ts +40 -9
  52. package/src/modes/acp/acp-agent.ts +564 -37
  53. package/src/modes/acp/terminal-auth.ts +2 -2
  54. package/src/modes/components/bash-execution.ts +14 -0
  55. package/src/modes/components/eval-execution.ts +14 -0
  56. package/src/modes/controllers/command-controller.ts +63 -8
  57. package/src/modes/utils/ui-helpers.ts +81 -11
  58. package/src/sdk/acp/adapter.ts +46 -45
  59. package/src/sdk/acp/index.ts +1 -1
  60. package/src/sdk/broker/broker.ts +392 -11
  61. package/src/sdk/broker/ensure.ts +131 -61
  62. package/src/sdk/broker/index.ts +1 -0
  63. package/src/sdk/broker/lifecycle-ledger.ts +59 -5
  64. package/src/sdk/broker/lifecycle.ts +278 -63
  65. package/src/sdk/broker/startup-budget.ts +103 -0
  66. package/src/sdk/bus/chat-daemon-control.ts +9 -3
  67. package/src/sdk/bus/chat-daemon-runtime.ts +565 -29
  68. package/src/sdk/bus/conversation-store.ts +1 -1
  69. package/src/sdk/bus/discord-daemon.ts +2 -1
  70. package/src/sdk/bus/discord-live-provider.ts +34 -6
  71. package/src/sdk/bus/index.ts +118 -20
  72. package/src/sdk/bus/kind-aware-reconciliation.ts +2 -1
  73. package/src/sdk/bus/prompt-reconciliation.ts +4 -9
  74. package/src/sdk/bus/slack-daemon.ts +91 -18
  75. package/src/sdk/cli/session-cli.ts +8 -1
  76. package/src/sdk/host/query/handlers.ts +1 -1
  77. package/src/sdk/host/query/revision-store.ts +48 -12
  78. package/src/sdk/host/session-runtime.test.ts +50 -0
  79. package/src/sdk/host/session-runtime.ts +12 -2
  80. package/src/sdk/lifecycle-session.ts +26 -3
  81. package/src/sdk/mcp/server.ts +6 -1
  82. package/src/sdk/prompt-failure.ts +42 -0
  83. package/src/sdk/prompt-watchdog.ts +162 -0
  84. package/src/sdk/session-reconnect.ts +43 -0
  85. package/src/sdk/startup-capability.ts +13 -4
  86. package/src/session/agent-session.ts +525 -222
  87. package/src/session/blob-store.ts +89 -0
  88. package/src/session/internal/managed-session-scope.ts +36 -26
  89. package/src/session/internal/managed-session-storage.ts +303 -32
  90. package/src/session/internal/session-memory-sidecar.ts +2160 -0
  91. package/src/session/session-manager.ts +7027 -394
  92. package/src/session/session-storage.ts +1134 -11
  93. package/src/task/executor.ts +17 -4
  94. package/src/tools/bash.ts +109 -7
  95. package/src/tools/find.ts +55 -26
  96. package/src/tools/todo-contract.ts +40 -8
  97. package/src/tui/output-block.ts +11 -1
@@ -0,0 +1,2160 @@
1
+ /**
2
+ * Standalone sidecar primitives for bounded-memory cold-session offloading.
3
+ *
4
+ * This module is intentionally self-contained: it imports only Node built-ins and
5
+ * defines the reusable P2/P3/P4 primitives agreed in the approved memory spec —
6
+ * the retained-byte accountant, reducer disk-ref demotion guardrails, the
7
+ * `isDerivedSessionMemoryFile` lifecycle predicate, the disposable sidecar v2
8
+ * index schemas, the unsorted append-only bounded dictionary builder with
9
+ * duplicate-ID sidecar-ineligibility detection, fixed-size cache accounting,
10
+ * the anchored base digest + rolling tail checksum chain with commit validation
11
+ * and the five-class reopen classifier, the `latestModelChange` / TTSR
12
+ * latest-wins reducer semantics, and bounded parent→children / labels/pins
13
+ * descriptors.
14
+ *
15
+ * Transcript v5 remains authoritative and every sidecar index is disposable /
16
+ * rebuildable. No builder here requires the whole transcript resident in
17
+ * memory; all builders consume records one at a time and enforce their peak
18
+ * budgets as they go.
19
+ */
20
+
21
+ import { createHash } from "node:crypto";
22
+
23
+ // ============================================================================
24
+ // Budgets and fixed-size bounds
25
+ // ============================================================================
26
+
27
+ /** Provider-context materialization acceptance peak (AC1). */
28
+ export const SESSION_MEMORY_ACCEPTANCE_BUDGET_BYTES = 64 * 1024 * 1024;
29
+ /** Steady-state retained-byte budget: 61.0625 MiB leaves the 64 MiB margin. */
30
+ export const SESSION_MEMORY_STEADY_STATE_BUDGET_BYTES = 61 * 1024 * 1024 + 64 * 1024;
31
+
32
+ /** Fixed 24 B resident disk-ref descriptor (ADR / A.1). */
33
+ export const DESCRIPTOR_BYTES = 24;
34
+ /** Conservative per-record object overhead (A.1). */
35
+ export const RECORD_OBJECT_OVERHEAD_BYTES = 48;
36
+ /** labels+pins steady-state budget (0.0625 MiB). */
37
+ export const LABELS_PINS_BUDGET_BYTES = 64 * 1024;
38
+
39
+ /** Reducer resident budget (4 MiB) — enforced by disk-ref demotion. */
40
+ export const REDUCER_BUDGET_BYTES = 4 * 1024 * 1024;
41
+ /** Guardrail: values over this are demoted to disk-ref descriptors. */
42
+ export const MAX_REDUCER_INLINE_BYTES = 4 * 1024;
43
+ /** Guardrail: max resident string length. */
44
+ export const MAX_REDUCER_STRING_BYTES = 4 * 1024;
45
+ /** Guardrail: max distinct roles. */
46
+ export const MAX_ROLES = 64;
47
+ /** Guardrail: max chain entries per role. */
48
+ export const MAX_CHAIN_ENTRIES = 32;
49
+ /** Guardrail: max TTSR rules. */
50
+ export const MAX_TTSR_RULES = 256;
51
+ /** Guardrail: max MCP names. */
52
+ export const MAX_MCP_NAMES = 4096;
53
+ /** Guardrail: max chars per MCP name. */
54
+ export const MAX_MCP_NAME_CHARS = 256;
55
+ /** Guardrail: modeData ≤ 64 KiB. */
56
+ export const MODE_DATA_MAX_BYTES = 64 * 1024;
57
+ /** Guardrail: compaction summary ≤ 64 KiB. */
58
+ export const COMPACTION_SUMMARY_MAX_BYTES = 64 * 1024;
59
+ /** preserveData.openaiRemoteCompaction ≤ 8 MiB (always a disk ref). */
60
+ export const OPENAI_REMOTE_COMPACTION_MAX_BYTES = 8 * 1024 * 1024;
61
+
62
+ /** Fixed-size index block cache (steady-state). */
63
+ export const BLOCK_CACHE_BUDGET_BYTES = 8 * 1024 * 1024;
64
+ /** Fixed-size resolved-entry cache (steady-state). */
65
+ export const ENTRY_CACHE_BUDGET_BYTES = 28 * 1024 * 1024;
66
+ /** Tail journal buffer/index (steady-state). */
67
+ export const TAIL_BUFFER_BUDGET_BYTES = 4 * 1024 * 1024;
68
+ /** Bounded forward-scan window for transcript-ahead recovery (R2.4). */
69
+ export const INDEX_TAIL_SCAN_MAX_BYTES = 4 * 1024 * 1024;
70
+ /** Maximum ordinal-index records retained transiently for one cold branch activation. */
71
+ export const COLD_BRANCH_PREFETCH_MAX_INDEX_ENTRIES = 16 * 1024;
72
+
73
+ /**
74
+ * Decide the cold-branch ordinal-run fast path before allocating its Map or
75
+ * transcript window. Large sparse/interleaved intervals fail closed to bounded
76
+ * dictionary/index fallback; the transcript remains authoritative.
77
+ */
78
+ export function coldBranchOrdinalRunWithinPrefetchBounds(input: {
79
+ boundaryOrdinal: number;
80
+ leafOrdinal: number;
81
+ transcriptStart: number;
82
+ transcriptEnd: number;
83
+ maxTranscriptBytes: number;
84
+ }): boolean {
85
+ const values = [
86
+ input.boundaryOrdinal,
87
+ input.leafOrdinal,
88
+ input.transcriptStart,
89
+ input.transcriptEnd,
90
+ input.maxTranscriptBytes,
91
+ ];
92
+ if (!values.every(value => Number.isSafeInteger(value))) return false;
93
+ const intervalEntries = input.leafOrdinal - input.boundaryOrdinal + 1;
94
+ const transcriptLength = input.transcriptEnd - input.transcriptStart;
95
+ return (
96
+ input.boundaryOrdinal >= 0 &&
97
+ intervalEntries > 0 &&
98
+ intervalEntries <= COLD_BRANCH_PREFETCH_MAX_INDEX_ENTRIES &&
99
+ input.transcriptStart >= 0 &&
100
+ transcriptLength > 0 &&
101
+ transcriptLength <= input.maxTranscriptBytes
102
+ );
103
+ }
104
+
105
+ /** Dictionary build acceptance cap; default buffers stay well below this 20 MiB maximum. */
106
+ export const DICTIONARY_BUILD_PEAK_BYTES = 20 * 1024 * 1024;
107
+ /** One dictionary partition buffer. */
108
+ export const DICTIONARY_PARTITION_BUFFER_BYTES = 128 * 1024;
109
+ /** Dictionary bucket journal flush buffer. */
110
+ export const DICTIONARY_BUCKET_JOURNAL_BYTES = 128 * 1024;
111
+
112
+ /** Bounded parent→children index bounds. */
113
+ export const PARENT_CHILDREN_MAX_PARENTS = 4096;
114
+ export const PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT = 256;
115
+ export const PARENT_CHILDREN_BUDGET_BYTES = 4 * 1024 * 1024;
116
+ /** Deterministic hash-bucket count for the disposable on-disk parent artifact. */
117
+ export const PARENT_CHILDREN_BUCKET_COUNT = 64;
118
+
119
+ // ============================================================================
120
+ // Retained-byte formulas (A.1)
121
+ // ============================================================================
122
+
123
+ /** Resident string = `2 × charCount + 16 B`. */
124
+ export function residentStringBytes(value: string): number {
125
+ return 2 * value.length + 16;
126
+ }
127
+
128
+ /** Resident array = `8 B/slot + Σ elements`. */
129
+ export function residentArrayBytes(slotCount: number, elementBytes: number): number {
130
+ return 8 * slotCount + elementBytes;
131
+ }
132
+
133
+ /** Per-record object with `childBytes` of payload = 48 B overhead + payload. */
134
+ export function residentRecordBytes(childBytes: number): number {
135
+ return RECORD_OBJECT_OVERHEAD_BYTES + childBytes;
136
+ }
137
+
138
+ /**
139
+ * Authoritative measured retained-byte counter. Enforcement acts on the total,
140
+ * never on field counts alone. `tryCharge`/`wouldExceed` let callers enforce a
141
+ * budget without mutating state (used by the provider-context preflight which
142
+ * must never retain an over-budget graph).
143
+ */
144
+ export class SessionMemoryAccountant {
145
+ private total = 0;
146
+ private readonly budget: number;
147
+
148
+ constructor(budgetBytes: number = SESSION_MEMORY_STEADY_STATE_BUDGET_BYTES) {
149
+ this.budget = budgetBytes;
150
+ }
151
+
152
+ get totalBytes(): number {
153
+ return this.total;
154
+ }
155
+
156
+ get budgetBytes(): number {
157
+ return this.budget;
158
+ }
159
+
160
+ get remainingBytes(): number {
161
+ return Math.max(0, this.budget - this.total);
162
+ }
163
+
164
+ /** Adds `bytes` to the running total and returns the new total. */
165
+ charge(bytes: number): number {
166
+ this.assertBytes(bytes);
167
+ this.total += bytes;
168
+ return this.total;
169
+ }
170
+
171
+ /** Charges one resident string via the `2 × charCount + 16` formula. */
172
+ chargeString(value: string): number {
173
+ return this.charge(residentStringBytes(value));
174
+ }
175
+
176
+ /** Charges one resident array via the `8 B/slot + Σ elements` formula. */
177
+ chargeArray(slotCount: number, elementBytes: number): number {
178
+ return this.charge(residentArrayBytes(slotCount, elementBytes));
179
+ }
180
+
181
+ /** Charges one fixed 24 B disk-ref descriptor. */
182
+ chargeDescriptor(): number {
183
+ return this.charge(DESCRIPTOR_BYTES);
184
+ }
185
+
186
+ /** Charges one per-record object (48 B overhead + payload). */
187
+ chargeRecord(childBytes: number): number {
188
+ return this.charge(residentRecordBytes(childBytes));
189
+ }
190
+
191
+ /**
192
+ * Adds `bytes` only if the running total stays within the budget.
193
+ * Returns `false` (and does not mutate) when the addition would exceed it.
194
+ */
195
+ tryCharge(bytes: number): boolean {
196
+ this.assertBytes(bytes);
197
+ if (this.total + bytes > this.budget) return false;
198
+ this.total += bytes;
199
+ return true;
200
+ }
201
+
202
+ /** Returns `true` when charging `bytes` would exceed the budget. */
203
+ wouldExceed(bytes: number): boolean {
204
+ this.assertBytes(bytes);
205
+ return this.total + bytes > this.budget;
206
+ }
207
+
208
+ isWithinBudget(): boolean {
209
+ return this.total <= this.budget;
210
+ }
211
+
212
+ release(bytes: number): void {
213
+ if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_release_bytes");
214
+ if (bytes > this.total) throw new RangeError("release_exceeds_charged");
215
+ this.total -= bytes;
216
+ }
217
+
218
+ snapshot(): { totalBytes: number; budgetBytes: number; remainingBytes: number } {
219
+ return { totalBytes: this.total, budgetBytes: this.budget, remainingBytes: this.remainingBytes };
220
+ }
221
+
222
+ private assertBytes(bytes: number): void {
223
+ if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_charge_bytes");
224
+ }
225
+ }
226
+
227
+ // ============================================================================
228
+ // Reducer disk-ref demotion guardrails
229
+ // ============================================================================
230
+
231
+ /**
232
+ * A 24 B resident descriptor that points at a value stored in the sidecar
233
+ * metadata-delta section. Demoted values are never resident as full strings.
234
+ */
235
+ export interface ReducerDemotionDescriptor {
236
+ section: "metadata-delta";
237
+ key: string;
238
+ offset: number;
239
+ length: number;
240
+ sha256: string;
241
+ }
242
+
243
+ export interface ReducerValueSnapshot {
244
+ key: string;
245
+ kind: "inline" | "disk_ref";
246
+ residentBytes: number;
247
+ storedBytes: number;
248
+ }
249
+
250
+ export type ReducerBudgetResult =
251
+ | { kind: "ok"; totalBytes: number }
252
+ | { kind: "over_budget_irreducible"; totalBytes: number };
253
+
254
+ interface TrackedReducerValue {
255
+ key: string;
256
+ kind: "inline" | "disk_ref";
257
+ residentBytes: number;
258
+ storedBytes: number;
259
+ sha256: string;
260
+ metadataBytes: number;
261
+ offset: number;
262
+ }
263
+
264
+ /**
265
+ * Tracks the reducer's resident bytes against the 4 MiB budget. Values over
266
+ * `MAX_REDUCER_INLINE_BYTES` are immediately stored as 24 B disk-ref
267
+ * descriptors. When the total would exceed the budget, the largest resident
268
+ * inline value is demoted, repeating until the total fits or only
269
+ * descriptors/scalars remain (impossible by arithmetic — checked anyway).
270
+ */
271
+ export class ReducerBudget {
272
+ private readonly values = new Map<string, TrackedReducerValue>();
273
+ private total = 0;
274
+ private nextOffset = 0;
275
+ private readonly budgetBytesValue: number;
276
+ private readonly maxInlineBytes: number;
277
+
278
+ constructor(budgetBytes: number = REDUCER_BUDGET_BYTES, maxInlineBytes: number = MAX_REDUCER_INLINE_BYTES) {
279
+ this.budgetBytesValue = budgetBytes;
280
+ this.maxInlineBytes = maxInlineBytes;
281
+ }
282
+
283
+ get totalBytes(): number {
284
+ return this.total;
285
+ }
286
+
287
+ get budgetBytes(): number {
288
+ return this.budgetBytesValue;
289
+ }
290
+
291
+ get(key: string): ReducerValueSnapshot | undefined {
292
+ const value = this.values.get(key);
293
+ if (value === undefined) return undefined;
294
+ return { key: value.key, kind: value.kind, residentBytes: value.residentBytes, storedBytes: value.storedBytes };
295
+ }
296
+
297
+ /** Returns the resident 24 B descriptor for a demoted key, if any. */
298
+ getDescriptor(key: string): ReducerDemotionDescriptor | undefined {
299
+ const value = this.values.get(key);
300
+ if (value === undefined || value.kind !== "disk_ref") return undefined;
301
+ return {
302
+ section: "metadata-delta",
303
+ key: value.key,
304
+ offset: value.offset,
305
+ length: value.storedBytes,
306
+ sha256: value.sha256,
307
+ };
308
+ }
309
+
310
+ /**
311
+ * Records a reducer value of `storedBytes` bytes (optionally with its content
312
+ * digest) and enforces the budget by demoting the largest resident inline
313
+ * value until the total fits.
314
+ */
315
+ setInline(key: string, storedBytes: number, sha256 = ""): ReducerBudgetResult {
316
+ if (!Number.isSafeInteger(storedBytes) || storedBytes < 0) throw new RangeError("invalid_reducer_value_bytes");
317
+ const previous = this.values.get(key);
318
+ if (previous !== undefined) {
319
+ this.total -= previous.residentBytes + previous.metadataBytes;
320
+ this.values.delete(key);
321
+ }
322
+ const metadataBytes = residentStringBytes(key) + residentStringBytes(sha256) + RECORD_OBJECT_OVERHEAD_BYTES;
323
+ if (storedBytes > this.maxInlineBytes) {
324
+ this.values.set(key, {
325
+ key,
326
+ kind: "disk_ref",
327
+ residentBytes: DESCRIPTOR_BYTES,
328
+ metadataBytes,
329
+ storedBytes,
330
+ sha256,
331
+ offset: this.nextOffset++,
332
+ });
333
+ this.total += DESCRIPTOR_BYTES + metadataBytes;
334
+ } else {
335
+ this.values.set(key, {
336
+ key,
337
+ kind: "inline",
338
+ residentBytes: storedBytes,
339
+ metadataBytes,
340
+ storedBytes,
341
+ sha256,
342
+ offset: this.nextOffset++,
343
+ });
344
+ this.total += storedBytes + metadataBytes;
345
+ }
346
+ return this.enforceBudget();
347
+ }
348
+
349
+ /** Demotes every currently resident inline value to a disk-ref descriptor. */
350
+ demoteAll(): void {
351
+ for (const value of this.values.values()) {
352
+ if (value.kind === "inline" && value.residentBytes > DESCRIPTOR_BYTES) this.demote(value);
353
+ }
354
+ }
355
+
356
+ private enforceBudget(): ReducerBudgetResult {
357
+ while (this.total > this.budgetBytesValue) {
358
+ const largest = this.largestDemotableInline();
359
+ if (largest === undefined) {
360
+ // Only descriptors/scalars remain but still over budget — impossible
361
+ // by arithmetic; the caller keeps the record hot and raises the
362
+ // token-compaction trigger. Never unbounded resident state.
363
+ return { kind: "over_budget_irreducible", totalBytes: this.total };
364
+ }
365
+ this.demote(largest);
366
+ }
367
+ return { kind: "ok", totalBytes: this.total };
368
+ }
369
+
370
+ private largestDemotableInline(): TrackedReducerValue | undefined {
371
+ let best: TrackedReducerValue | undefined;
372
+ for (const value of this.values.values()) {
373
+ if (
374
+ value.kind === "inline" &&
375
+ value.residentBytes > DESCRIPTOR_BYTES &&
376
+ (best === undefined || value.residentBytes > best.residentBytes)
377
+ ) {
378
+ best = value;
379
+ }
380
+ }
381
+ return best;
382
+ }
383
+
384
+ private demote(value: TrackedReducerValue): void {
385
+ const delta = value.residentBytes - DESCRIPTOR_BYTES;
386
+ value.residentBytes = DESCRIPTOR_BYTES;
387
+ value.kind = "disk_ref";
388
+ this.total -= delta;
389
+ }
390
+ }
391
+
392
+ // ============================================================================
393
+ // Derived-file lifecycle predicate (MEM-C-014/024)
394
+ // ============================================================================
395
+
396
+ const SPILL_SEGMENT = ".spill.";
397
+ const DERIVED_SUFFIXES = new Set(["idx", "tail", "commit", "buckets"]);
398
+ const DERIVED_PREFIXES = ["dict-", "capture-", "fork-", "overlay-", "parent-", "metadata-"];
399
+
400
+ /**
401
+ * Single source of truth for every sidecar artifact and temp: `*.spill.idx`,
402
+ * `*.spill.tail`, `*.spill.commit`, `*.spill.buckets`, `*.spill.dict-*`,
403
+ * `*.spill.capture-*`, `*.spill.fork-*`, `*.spill.overlay-*`,
404
+ * `*.spill.parent-*` (persistent parent→children buckets),
405
+ * `*.spill.metadata-*` (persistent metadata-delta section), and any
406
+ * `*.spill.*.tmp`. Used at every fork/delete/sweep/Memory-parity callsite so
407
+ * ad-hoc filters cannot drift.
408
+ */
409
+ export function isDerivedSessionMemoryFile(pathname: string): boolean {
410
+ const base = pathname.slice(Math.max(pathname.lastIndexOf("/"), pathname.lastIndexOf("\\")) + 1);
411
+ const idx = base.indexOf(SPILL_SEGMENT);
412
+ if (idx === -1) return false;
413
+ const rest = base.slice(idx + SPILL_SEGMENT.length);
414
+ if (rest.length === 0) return false;
415
+ if (DERIVED_SUFFIXES.has(rest)) return true;
416
+ for (const prefix of DERIVED_PREFIXES) {
417
+ if (rest.startsWith(prefix) && rest.length > prefix.length) return true;
418
+ }
419
+ if (rest.endsWith(".tmp") && rest.length > ".tmp".length) return true;
420
+ return false;
421
+ }
422
+
423
+ // ============================================================================
424
+ // Sidecar v2 disposable index schemas
425
+ // ============================================================================
426
+
427
+ /** Common header for every sidecar v2 index file. */
428
+ export interface SidecarIndexHeaderV2 {
429
+ version: 2;
430
+ sessionId: string;
431
+ /** Set when duplicate record IDs were detected; session is eager-ineligible. */
432
+ sidecarIneligible: boolean;
433
+ }
434
+
435
+ /** Unsorted append-only variable-width dictionary (term → id). */
436
+ export interface SidecarDictionaryV2 {
437
+ header: SidecarIndexHeaderV2;
438
+ /** Insertion-ordered unique terms (append-only; deterministic). */
439
+ terms: readonly string[];
440
+ /** term → dictionary id, assigned by first-insertion order. */
441
+ idByTerm: ReadonlyMap<string, number>;
442
+ /** Combined serialized term bytes. */
443
+ totalBytes: number;
444
+ }
445
+
446
+ /** Rolling tail checksum index (anchored base + terminal chain). */
447
+ export interface SidecarTailIndexV2 {
448
+ header: SidecarIndexHeaderV2;
449
+ base: BaseAnchor;
450
+ records: readonly TailRecord[];
451
+ terminalChecksum: string;
452
+ terminalSeq: number;
453
+ transcriptSize: number;
454
+ }
455
+
456
+ /** Metadata-delta section: demoted reducer values + deltas. */
457
+ export interface SidecarMetadataIndexV2 {
458
+ header: SidecarIndexHeaderV2;
459
+ /** key → { offset, length, sha256 } for demoted values. */
460
+ entries: Record<string, { offset: number; length: number; sha256: string }>;
461
+ }
462
+
463
+ /** Bounded parent→children index. */
464
+ export interface SidecarParentIndexV2 {
465
+ header: SidecarIndexHeaderV2;
466
+ entries: readonly ParentChildrenIndexEntry[];
467
+ }
468
+
469
+ /** Bounded labels/pins descriptor. */
470
+ export interface SidecarLabelsPinsIndexV2 {
471
+ header: SidecarIndexHeaderV2;
472
+ labels: ReadonlyMap<string, string>;
473
+ pins: ReadonlyMap<string, string>;
474
+ totalBytes: number;
475
+ }
476
+
477
+ // ============================================================================
478
+ // Bounded dictionary builder (20 MiB peak, duplicate-ID detection)
479
+ // ============================================================================
480
+
481
+ export interface DictionaryRecordInput {
482
+ ordinal: number;
483
+ id: string;
484
+ bytes: Uint8Array;
485
+ }
486
+
487
+ export interface DictionaryBuildOptions {
488
+ peakBudgetBytes?: number;
489
+ partitionBufferBytes?: number;
490
+ bucketJournalBytes?: number;
491
+ }
492
+
493
+ export type DictionaryAddResult = { kind: "ok" } | { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number };
494
+
495
+ export type DictionaryBuildResult =
496
+ | { kind: "ok"; dictionary: SidecarDictionaryV2; stats: DictionaryBuildStats }
497
+ | { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number };
498
+
499
+ export interface DictionaryBuildStats {
500
+ totalRecords: number;
501
+ uniqueTerms: number;
502
+ totalBytes: number;
503
+ peakBytes: number;
504
+ duplicateIds: readonly string[];
505
+ sidecarIneligible: boolean;
506
+ }
507
+
508
+ const utf8Decoder = new TextDecoder("utf-8");
509
+
510
+ /**
511
+ * Unsorted append-only variable-width dictionary builder. Consumes records one
512
+ * at a time (no whole transcript resident) and tracks the 20 MiB build peak as
513
+ * `Σ term bytes + 4 × partition buffer + bucket journal`. Duplicate record IDs
514
+ * mark the session sidecar-ineligible (eager fallback) while the build still
515
+ * completes deterministically.
516
+ */
517
+ export class BoundedDictionaryBuilder {
518
+ private readonly terms: string[] = [];
519
+ private readonly idByTerm = new Map<string, number>();
520
+ private readonly seenRecordIds = new Map<string, number>();
521
+ private readonly duplicateIds: string[] = [];
522
+ private totalTermBytes = 0;
523
+ private recordIdBytes = 0;
524
+ private duplicateIdBytes = 0;
525
+ private peakBytes = 0;
526
+ private sidecarIneligible = false;
527
+ private recordCount = 0;
528
+ private finished = false;
529
+ private readonly peakBudget: number;
530
+ private readonly partitionBase: number;
531
+
532
+ constructor(options: DictionaryBuildOptions = {}) {
533
+ this.peakBudget = options.peakBudgetBytes ?? DICTIONARY_BUILD_PEAK_BYTES;
534
+ const partition = options.partitionBufferBytes ?? DICTIONARY_PARTITION_BUFFER_BYTES;
535
+ const journal = options.bucketJournalBytes ?? DICTIONARY_BUCKET_JOURNAL_BYTES;
536
+ this.partitionBase = 4 * partition + journal;
537
+ }
538
+
539
+ add(record: DictionaryRecordInput): DictionaryAddResult {
540
+ if (this.finished) throw new Error("dictionary_builder_finished");
541
+ this.recordCount += 1;
542
+ const priorOrdinal = this.seenRecordIds.get(record.id);
543
+ if (priorOrdinal !== undefined) {
544
+ this.sidecarIneligible = true;
545
+ if (!this.duplicateIds.includes(record.id)) {
546
+ this.duplicateIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
547
+ this.duplicateIds.push(record.id);
548
+ }
549
+ } else {
550
+ this.recordIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
551
+ this.seenRecordIds.set(record.id, record.ordinal);
552
+ }
553
+ this.updatePeak();
554
+ if (this.peakBytes > this.peakBudget) {
555
+ return { kind: "budget_exceeded", peakBytes: this.peakBytes, budgetBytes: this.peakBudget };
556
+ }
557
+ const term = utf8Decoder.decode(record.bytes);
558
+ if (!this.idByTerm.has(term)) {
559
+ this.idByTerm.set(term, this.idByTerm.size);
560
+ this.terms.push(term);
561
+ this.totalTermBytes += residentStringBytes(term) + RECORD_OBJECT_OVERHEAD_BYTES;
562
+ this.updatePeak();
563
+ }
564
+ if (this.peakBytes > this.peakBudget) {
565
+ return { kind: "budget_exceeded", peakBytes: this.peakBytes, budgetBytes: this.peakBudget };
566
+ }
567
+ return { kind: "ok" };
568
+ }
569
+
570
+ finish(): DictionaryBuildResult {
571
+ if (this.finished) throw new Error("dictionary_builder_already_finished");
572
+ this.finished = true;
573
+ return {
574
+ kind: "ok",
575
+ dictionary: {
576
+ header: { version: 2, sessionId: "", sidecarIneligible: this.sidecarIneligible },
577
+ terms: this.terms,
578
+ idByTerm: this.idByTerm,
579
+ totalBytes: this.totalTermBytes,
580
+ },
581
+ stats: {
582
+ totalRecords: this.recordCount,
583
+ uniqueTerms: this.terms.length,
584
+ totalBytes: this.totalTermBytes,
585
+ peakBytes: this.peakBytes,
586
+ duplicateIds: this.duplicateIds,
587
+ sidecarIneligible: this.sidecarIneligible,
588
+ },
589
+ };
590
+ }
591
+
592
+ private updatePeak(): void {
593
+ const current = Math.max(this.partitionBase, this.totalTermBytes + this.recordIdBytes + this.duplicateIdBytes);
594
+ if (current > this.peakBytes) this.peakBytes = current;
595
+ }
596
+ }
597
+
598
+ // ============================================================================
599
+ // Fixed-size cache accounting (block / entry / tail)
600
+ // ============================================================================
601
+
602
+ /**
603
+ * Tracks a fixed-size cache budget (8 MiB block, 28 MiB entry, 4 MiB tail).
604
+ * `tryAllocate` rejects allocations that would exceed the budget without
605
+ * mutating state.
606
+ */
607
+ export class FixedCacheAccount {
608
+ private allocated = 0;
609
+ private readonly budget: number;
610
+
611
+ constructor(budgetBytes: number) {
612
+ this.budget = budgetBytes;
613
+ }
614
+
615
+ get allocatedBytes(): number {
616
+ return this.allocated;
617
+ }
618
+
619
+ get budgetBytes(): number {
620
+ return this.budget;
621
+ }
622
+
623
+ get remainingBytes(): number {
624
+ return Math.max(0, this.budget - this.allocated);
625
+ }
626
+
627
+ tryAllocate(bytes: number): boolean {
628
+ if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_cache_allocation");
629
+ if (this.allocated + bytes > this.budget) return false;
630
+ this.allocated += bytes;
631
+ return true;
632
+ }
633
+
634
+ release(bytes: number): void {
635
+ if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_cache_release");
636
+ if (bytes > this.allocated) throw new RangeError("cache_release_exceeds_allocated");
637
+ this.allocated -= bytes;
638
+ }
639
+ }
640
+
641
+ // ============================================================================
642
+ // Anchored base digest + rolling tail checksum chain (C.1)
643
+ // ============================================================================
644
+
645
+ /** Anchors the base range `[0, baseEndOffset)` with a content digest. */
646
+ export interface BaseAnchor {
647
+ baseDigest: string;
648
+ baseEndOffset: number;
649
+ }
650
+
651
+ export type TailRecordKind = "user" | "assistant" | "tool" | "model_change" | "ttsr_injection" | "internal" | "other";
652
+
653
+ /** One committed rolling tail record. */
654
+ export interface TailRecord {
655
+ gen: number;
656
+ seq: number;
657
+ kind: TailRecordKind;
658
+ ordinal: number;
659
+ id: string;
660
+ parentId: string | null;
661
+ type: string;
662
+ byteOffset: number;
663
+ byteLength: number;
664
+ recordDigest: string;
665
+ checksum: string;
666
+ }
667
+
668
+ /** Input for appending to a rolling tail chain (checksum is computed). */
669
+ export interface TailRecordInput {
670
+ gen?: number;
671
+ seq: number;
672
+ kind: TailRecordKind;
673
+ ordinal: number;
674
+ id: string;
675
+ parentId: string | null;
676
+ type: string;
677
+ byteOffset: number;
678
+ byteLength: number;
679
+ recordDigest: string;
680
+ }
681
+
682
+ /** Effective base+tail identity (no per-append whole-file hash). */
683
+ export interface CommittedTail {
684
+ base: BaseAnchor;
685
+ records: readonly TailRecord[];
686
+ terminalChecksum: string;
687
+ terminalSeq: number;
688
+ transcriptSize: number;
689
+ }
690
+
691
+ /** SHA-256 of one record line's bytes. */
692
+ export function computeLineDigest(bytes: Uint8Array): string {
693
+ return createHash("sha256").update(bytes).digest("hex");
694
+ }
695
+
696
+ /**
697
+ * Canonical length-prefixed chain hash. Each part is hashed as its UTF-8 /
698
+ * raw bytes prefixed by a 4-byte big-endian length, so concatenations are
699
+ * unambiguous.
700
+ */
701
+ export function hashChain(parts: ReadonlyArray<bigint | number | string | Uint8Array>): string {
702
+ const hash = createHash("sha256");
703
+ for (const part of parts) {
704
+ const bytes =
705
+ typeof part === "string"
706
+ ? Buffer.from(part, "utf8")
707
+ : typeof part === "number"
708
+ ? Buffer.from(String(part), "utf8")
709
+ : typeof part === "bigint"
710
+ ? Buffer.from(part.toString(), "utf8")
711
+ : Buffer.from(part);
712
+ const length = Buffer.allocUnsafe(4);
713
+ length.writeUInt32BE(bytes.byteLength, 0);
714
+ hash.update(length).update(bytes);
715
+ }
716
+ return hash.digest("hex");
717
+ }
718
+
719
+ /** First tail checksum: `C0 = H(baseDigest ‖ baseEndOffset ‖ seq0 ‖ lineDigest0)`. */
720
+ export function computeC0(base: BaseAnchor, seq0: number, lineDigest0: string): string {
721
+ return hashChain([base.baseDigest, base.baseEndOffset, seq0, lineDigest0]);
722
+ }
723
+
724
+ /** Checksum one complete tail record, including every field used for ordering or lookup. */
725
+ export function computeTailRecordChecksum(
726
+ base: BaseAnchor,
727
+ previousChecksum: string | undefined,
728
+ record: TailRecordInput & { gen?: number },
729
+ ): string {
730
+ const fields: Array<string | number> = [
731
+ record.gen ?? 0,
732
+ record.seq,
733
+ record.kind,
734
+ record.ordinal,
735
+ record.id,
736
+ record.parentId === null ? "<null>" : record.parentId,
737
+ record.type,
738
+ record.byteOffset,
739
+ record.byteLength,
740
+ record.recordDigest,
741
+ ];
742
+ return previousChecksum === undefined
743
+ ? hashChain([base.baseDigest, base.baseEndOffset, ...fields])
744
+ : hashChain([previousChecksum, ...fields]);
745
+ }
746
+
747
+ /** Resident bytes for every retained tail string plus the record object. */
748
+ export function tailRecordResidentBytes(record: TailRecordInput, checksumChars = 64): number {
749
+ return residentRecordBytes(
750
+ residentStringBytes(record.id) +
751
+ (record.parentId === null ? 0 : residentStringBytes(record.parentId)) +
752
+ residentStringBytes(record.type) +
753
+ residentStringBytes(record.kind) +
754
+ residentStringBytes(record.recordDigest) +
755
+ (2 * checksumChars + 16),
756
+ );
757
+ }
758
+
759
+ /** Terminal checksum for a committed tail (empty chain → base only). */
760
+ export function computeTerminalChecksum(base: BaseAnchor, records: readonly TailRecord[]): string {
761
+ if (records.length === 0) return hashChain([base.baseDigest, base.baseEndOffset]);
762
+ return records[records.length - 1].checksum;
763
+ }
764
+
765
+ export interface TailValidationResult {
766
+ valid: boolean;
767
+ firstInvalidSeq?: number;
768
+ reason?: string;
769
+ }
770
+
771
+ /**
772
+ * Validates a committed tail chain: contiguous seq from 0, contiguous absolute
773
+ * offsets from `baseEndOffset`, and checksum continuity
774
+ * `checksum_i = H(checksum_{i-1} ‖ seq_i ‖ offset_i ‖ length_i ‖ lineDigest_i)`.
775
+ */
776
+ export function validateTailChain(base: BaseAnchor, records: readonly TailRecord[]): TailValidationResult {
777
+ let expectedSeq = 0;
778
+ let expectedOffset = base.baseEndOffset;
779
+ let previousChecksum: string | undefined;
780
+ for (const record of records) {
781
+ if (record.seq !== expectedSeq) {
782
+ return { valid: false, firstInvalidSeq: record.seq, reason: "seq_discontinuity" };
783
+ }
784
+ if (record.byteOffset !== expectedOffset) {
785
+ return { valid: false, firstInvalidSeq: record.seq, reason: "offset_discontinuity" };
786
+ }
787
+ const expectedChecksum = computeTailRecordChecksum(base, previousChecksum, record);
788
+ if (record.checksum !== expectedChecksum) {
789
+ return { valid: false, firstInvalidSeq: record.seq, reason: "checksum_mismatch" };
790
+ }
791
+ previousChecksum = expectedChecksum;
792
+ expectedSeq += 1;
793
+ expectedOffset = record.byteOffset + record.byteLength;
794
+ }
795
+ return { valid: true };
796
+ }
797
+
798
+ /**
799
+ * Streams tail records, computing the rolling checksum chain incrementally and
800
+ * enforcing the 4 MiB tail buffer budget. Never buffers the whole transcript.
801
+ */
802
+ export class RollingTailChainBuilder {
803
+ private readonly records: TailRecord[] = [];
804
+ private readonly base: BaseAnchor;
805
+ private lastChecksum: string | undefined;
806
+ private lastSeq = -1;
807
+ private lastEndOffset: number;
808
+ private remainingTailBytes: number;
809
+
810
+ constructor(base: BaseAnchor, options: { tailBufferBytes?: number } = {}) {
811
+ this.base = base;
812
+ this.lastEndOffset = base.baseEndOffset;
813
+ this.remainingTailBytes = options.tailBufferBytes ?? TAIL_BUFFER_BUDGET_BYTES;
814
+ }
815
+
816
+ get recordCount(): number {
817
+ return this.records.length;
818
+ }
819
+
820
+ /** Appends one tail record; returns `undefined` when the buffer budget is hit. */
821
+ append(input: TailRecordInput): TailRecord | undefined {
822
+ const expectedSeq = this.lastSeq + 1;
823
+ if (input.seq !== expectedSeq) throw new Error("tail_seq_discontinuity");
824
+ if (input.byteOffset !== this.lastEndOffset) throw new Error("tail_offset_discontinuity");
825
+ const checksum = computeTailRecordChecksum(this.base, this.lastChecksum, input);
826
+ const residentBytes = tailRecordResidentBytes(input, checksum.length);
827
+ if (residentBytes > this.remainingTailBytes) return undefined;
828
+ this.remainingTailBytes -= residentBytes;
829
+ const record: TailRecord = { ...input, gen: input.gen ?? 0, checksum };
830
+ this.records.push(record);
831
+ this.lastSeq = input.seq;
832
+ this.lastEndOffset = input.byteOffset + input.byteLength;
833
+ this.lastChecksum = checksum;
834
+ return record;
835
+ }
836
+
837
+ build(): CommittedTail {
838
+ return {
839
+ base: this.base,
840
+ records: this.records,
841
+ terminalChecksum: computeTerminalChecksum(this.base, this.records),
842
+ terminalSeq: this.lastSeq,
843
+ transcriptSize: this.lastEndOffset,
844
+ };
845
+ }
846
+ }
847
+
848
+ // ============================================================================
849
+ // Commit validation + five-class reopen classifier (R2.4)
850
+ // ============================================================================
851
+
852
+ /** Descriptor snapshot captured post-fsync from the authoritative open handle. */
853
+ export interface DescriptorSnapshot {
854
+ dev: bigint;
855
+ ino: bigint;
856
+ nlink?: bigint;
857
+ size: number;
858
+ mtimeNs: bigint;
859
+ ctimeNs: bigint;
860
+ }
861
+
862
+ /** Exact enum-order descriptor comparison (dev/ino/nlink/size/mtimeNs/ctimeNs). */
863
+ export function sameDescriptor(left: DescriptorSnapshot, right: DescriptorSnapshot): boolean {
864
+ return (
865
+ left.dev === right.dev &&
866
+ left.ino === right.ino &&
867
+ (left.nlink ?? 0n) === (right.nlink ?? 0n) &&
868
+ left.size === right.size &&
869
+ left.mtimeNs === right.mtimeNs &&
870
+ left.ctimeNs === right.ctimeNs
871
+ );
872
+ }
873
+
874
+ /** Contents of the mutable `.spill.commit` marker (R2). */
875
+ export interface CommitMarkerContents {
876
+ gen: number;
877
+ descriptor: DescriptorSnapshot;
878
+ base: BaseAnchor;
879
+ terminalChecksum: string;
880
+ terminalSeq: number;
881
+ transcriptSize: number;
882
+ }
883
+
884
+ export type CommitInvalidReason =
885
+ | "missing_fields"
886
+ | "base_invalid"
887
+ | "chain_invalid"
888
+ | "terminal_checksum_mismatch"
889
+ | "terminal_seq_mismatch"
890
+ | "transcript_size_mismatch"
891
+ | "descriptor_mismatch"
892
+ | "terminal_marker_mismatch";
893
+
894
+ export type CommitValidationResult = { kind: "valid" } | { kind: "invalid"; reason: CommitInvalidReason };
895
+
896
+ /**
897
+ * Validates a commit marker's internal consistency plus its descriptor against
898
+ * the currently opened transcript. `baseValid`/`tailValid`/`terminalMarkerValid`
899
+ * are the externally-observed proof results.
900
+ */
901
+ export function validateCommit(
902
+ commit: Partial<CommitMarkerContents> | CommitMarkerContents,
903
+ records: readonly TailRecord[],
904
+ observed: { descriptor: DescriptorSnapshot; baseValid: boolean; tailValid: boolean; terminalMarkerValid: boolean },
905
+ ): CommitValidationResult {
906
+ if (commit === null || typeof commit !== "object") return { kind: "invalid", reason: "missing_fields" };
907
+ if (
908
+ typeof commit.gen !== "number" ||
909
+ commit.descriptor === undefined ||
910
+ commit.base === undefined ||
911
+ typeof commit.terminalChecksum !== "string" ||
912
+ typeof commit.terminalSeq !== "number" ||
913
+ typeof commit.transcriptSize !== "number"
914
+ ) {
915
+ return { kind: "invalid", reason: "missing_fields" };
916
+ }
917
+ if (!observed.baseValid) return { kind: "invalid", reason: "base_invalid" };
918
+ const chain = validateTailChain(commit.base, records);
919
+ if (!chain.valid) return { kind: "invalid", reason: "chain_invalid" };
920
+ const computedTerminal = computeTerminalChecksum(commit.base, records);
921
+ if (commit.terminalChecksum !== computedTerminal) return { kind: "invalid", reason: "terminal_checksum_mismatch" };
922
+ const expectedTerminalSeq = records.length === 0 ? -1 : records[records.length - 1].seq;
923
+ if (commit.terminalSeq !== expectedTerminalSeq) return { kind: "invalid", reason: "terminal_seq_mismatch" };
924
+ const expectedSize =
925
+ records.length === 0
926
+ ? commit.base.baseEndOffset
927
+ : records[records.length - 1].byteOffset + records[records.length - 1].byteLength;
928
+ if (commit.transcriptSize !== expectedSize) return { kind: "invalid", reason: "transcript_size_mismatch" };
929
+ if (!sameDescriptor(commit.descriptor, observed.descriptor))
930
+ return { kind: "invalid", reason: "descriptor_mismatch" };
931
+ if (!observed.terminalMarkerValid) return { kind: "invalid", reason: "terminal_marker_mismatch" };
932
+ return { kind: "valid" };
933
+ }
934
+
935
+ export type ReopenClassKind = "exact" | "transcript_ahead" | "tail_ahead" | "rebuild" | "stale_commit";
936
+
937
+ /** Physical evidence the reopen classifier consumes. */
938
+ export interface ReopenEvidence {
939
+ markerPresent: boolean;
940
+ descriptorExact: boolean;
941
+ sameObject: boolean;
942
+ sameSize: boolean;
943
+ sizeGrew: boolean;
944
+ sizeShrank: boolean;
945
+ withinScanWindow: boolean;
946
+ timesAdvanced: boolean;
947
+ timesChanged: boolean;
948
+ baseValid: boolean;
949
+ tailValid: boolean;
950
+ terminalMarkerValid: boolean;
951
+ }
952
+
953
+ export interface ReopenClassification {
954
+ kind: ReopenClassKind;
955
+ reason: string;
956
+ }
957
+
958
+ /**
959
+ * Five-class reopen classification (R2.4). Deterministic over the supplied
960
+ * evidence; a stale commit is never accepted as current on any path.
961
+ */
962
+ export function classifyReopen(evidence: ReopenEvidence): ReopenClassification {
963
+ if (!evidence.markerPresent) return { kind: "stale_commit", reason: "no_commit_marker" };
964
+ if (evidence.descriptorExact) {
965
+ if (evidence.baseValid && evidence.tailValid && evidence.terminalMarkerValid) {
966
+ return { kind: "exact", reason: "descriptor_and_proof_match" };
967
+ }
968
+ return { kind: "rebuild", reason: "descriptor_exact_proof_invalid" };
969
+ }
970
+ if (evidence.sameObject && evidence.sameSize && evidence.timesChanged) {
971
+ return { kind: "rebuild", reason: "same_size_mutation" };
972
+ }
973
+ if (evidence.sameObject && evidence.sameSize) {
974
+ if (evidence.baseValid && !evidence.tailValid)
975
+ return { kind: "tail_ahead", reason: "committed_tail_record_invalid" };
976
+ if (evidence.baseValid && evidence.tailValid && evidence.terminalMarkerValid) {
977
+ return { kind: "exact", reason: "descriptor_equivalent" };
978
+ }
979
+ return { kind: "rebuild", reason: "proof_invalid" };
980
+ }
981
+ if (evidence.sameObject && evidence.sizeGrew) {
982
+ if (evidence.withinScanWindow && evidence.timesAdvanced && evidence.baseValid && evidence.tailValid) {
983
+ return { kind: "transcript_ahead", reason: "object_grew_within_window" };
984
+ }
985
+ return { kind: "rebuild", reason: "over_window_or_invalid_base" };
986
+ }
987
+ if (evidence.sameObject && evidence.sizeShrank) {
988
+ return { kind: "rebuild", reason: "shrink_outside_tail_state" };
989
+ }
990
+ return { kind: "stale_commit", reason: "object_identity_mismatch" };
991
+ }
992
+
993
+ // ============================================================================
994
+ // Reducer deltas: nearest model-change role (R1) + TTSR latest-wins
995
+ // ============================================================================
996
+
997
+ export interface LatestModelChangeDelta {
998
+ kind: "latest_model_change";
999
+ ordinal: number;
1000
+ role?: string;
1001
+ }
1002
+
1003
+ export interface TtsrDelta {
1004
+ kind: "ttsr_injection";
1005
+ ordinal: number;
1006
+ rulesCount: number;
1007
+ recordsCount: number;
1008
+ count: number;
1009
+ }
1010
+
1011
+ export type ReducerDelta = LatestModelChangeDelta | TtsrDelta;
1012
+
1013
+ export interface ModelChangeReducerState {
1014
+ latest: { ordinal: number; role: string | undefined } | undefined;
1015
+ }
1016
+
1017
+ export interface TtsrReducerState {
1018
+ count: number;
1019
+ rulesCount: number;
1020
+ recordsCount: number;
1021
+ largestOrdinal: number;
1022
+ }
1023
+
1024
+ export interface ReducerState {
1025
+ modelChange: ModelChangeReducerState;
1026
+ ttsr: TtsrReducerState;
1027
+ }
1028
+
1029
+ export function createReducerState(): ReducerState {
1030
+ return {
1031
+ modelChange: { latest: undefined },
1032
+ ttsr: { count: 0, rulesCount: 0, recordsCount: 0, largestOrdinal: -1 },
1033
+ };
1034
+ }
1035
+
1036
+ /**
1037
+ * Applies one reducer delta. `latest_model_change` overwrites the nearest
1038
+ * event's `{ordinal, role}`; `ttsr_injection.count` replaces the prior value
1039
+ * (authoritative `buildSessionContext`, latest-wins).
1040
+ */
1041
+ export function applyReducerDelta(state: ReducerState, delta: ReducerDelta): ReducerState {
1042
+ switch (delta.kind) {
1043
+ case "latest_model_change":
1044
+ return { ...state, modelChange: { latest: { ordinal: delta.ordinal, role: delta.role } } };
1045
+ case "ttsr_injection":
1046
+ return {
1047
+ ...state,
1048
+ ttsr: {
1049
+ count: delta.count,
1050
+ rulesCount: delta.rulesCount,
1051
+ recordsCount: delta.recordsCount,
1052
+ largestOrdinal: Math.max(state.ttsr.largestOrdinal, delta.ordinal),
1053
+ },
1054
+ };
1055
+ }
1056
+ }
1057
+
1058
+ /**
1059
+ * R1 consumer: returns `latestModelChange.role ?? "default"` when a
1060
+ * `model_change` exists on the path, else `undefined`. `hasExplicitDefaultModel`
1061
+ * only gates legacy assistant inference into `models.default`; it never feeds
1062
+ * this getter.
1063
+ */
1064
+ export function getLastModelChangeRole(state: ReducerState): string | undefined {
1065
+ const latest = state.modelChange.latest;
1066
+ if (latest === undefined) return undefined;
1067
+ return latest.role ?? "default";
1068
+ }
1069
+
1070
+ /**
1071
+ * Compaction fold: keeps the max-ordinal `model_change` (nearest wins) and the
1072
+ * latest-ordinal TTSR state, matching the eager leaf→root first-hit semantics.
1073
+ */
1074
+ export function foldReducerStates(left: ReducerState, right: ReducerState): ReducerState {
1075
+ const modelChange = foldModelChange(left.modelChange.latest, right.modelChange.latest);
1076
+ const ttsr = left.ttsr.largestOrdinal >= right.ttsr.largestOrdinal ? left.ttsr : right.ttsr;
1077
+ return { modelChange: { latest: modelChange }, ttsr };
1078
+ }
1079
+
1080
+ function foldModelChange(
1081
+ left: ModelChangeReducerState["latest"],
1082
+ right: ModelChangeReducerState["latest"],
1083
+ ): ModelChangeReducerState["latest"] {
1084
+ if (left === undefined) return right;
1085
+ if (right === undefined) return left;
1086
+ return left.ordinal >= right.ordinal ? left : right;
1087
+ }
1088
+
1089
+ // ============================================================================
1090
+ // Bounded parent→children and labels/pins descriptors
1091
+ // ============================================================================
1092
+
1093
+ export interface ParentChildrenIndexEntry {
1094
+ parentId: string;
1095
+ children: readonly string[];
1096
+ }
1097
+
1098
+ /** Bounded parent→children index (rejects when either bound is exceeded). */
1099
+ export class BoundedParentChildrenIndex {
1100
+ private readonly childrenByParent = new Map<string, string[]>();
1101
+ private readonly maxParents: number;
1102
+ private readonly maxChildrenPerParent: number;
1103
+ private readonly budgetBytesValue: number;
1104
+ private total = 0;
1105
+
1106
+ constructor(options: { maxParents?: number; maxChildrenPerParent?: number; budgetBytes?: number } = {}) {
1107
+ this.maxParents = options.maxParents ?? PARENT_CHILDREN_MAX_PARENTS;
1108
+ this.maxChildrenPerParent = options.maxChildrenPerParent ?? PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT;
1109
+ this.budgetBytesValue = options.budgetBytes ?? PARENT_CHILDREN_BUDGET_BYTES;
1110
+ }
1111
+
1112
+ get size(): number {
1113
+ return this.childrenByParent.size;
1114
+ }
1115
+
1116
+ get totalBytes(): number {
1117
+ return this.total;
1118
+ }
1119
+
1120
+ get budgetBytes(): number {
1121
+ return this.budgetBytesValue;
1122
+ }
1123
+
1124
+ add(parentId: string, childId: string): boolean {
1125
+ if (parentId.length === 0 || childId.length === 0) throw new Error("parent_children_empty_id");
1126
+ let children = this.childrenByParent.get(parentId);
1127
+ if (children?.includes(childId)) return true;
1128
+ const parentBytes = children === undefined ? residentStringBytes(parentId) + RECORD_OBJECT_OVERHEAD_BYTES : 0;
1129
+ const childBytes = residentStringBytes(childId) + 8;
1130
+ if (this.total + parentBytes + childBytes > this.budgetBytesValue) return false;
1131
+ if (children === undefined) {
1132
+ if (this.childrenByParent.size >= this.maxParents) return false;
1133
+ children = [];
1134
+ this.childrenByParent.set(parentId, children);
1135
+ }
1136
+ if (children.length >= this.maxChildrenPerParent) return false;
1137
+ children.push(childId);
1138
+ this.total += parentBytes + childBytes;
1139
+ return true;
1140
+ }
1141
+
1142
+ get(parentId: string): readonly string[] | undefined {
1143
+ const children = this.childrenByParent.get(parentId);
1144
+ return children ? [...children] : undefined;
1145
+ }
1146
+
1147
+ entries(): ParentChildrenIndexEntry[] {
1148
+ const out: ParentChildrenIndexEntry[] = [];
1149
+ for (const [parentId, children] of this.childrenByParent) out.push({ parentId, children: [...children] });
1150
+ return out;
1151
+ }
1152
+ }
1153
+
1154
+ /**
1155
+ * Bounded labels/pins store. Accounting uses `stored bytes + map overhead`
1156
+ * (resident string formulas + per-entry overhead), rejected when the labels+pins
1157
+ * budget would be exceeded.
1158
+ */
1159
+ export class BoundedLabelsPinsStore {
1160
+ private readonly labels = new Map<string, string>();
1161
+ private readonly pins = new Map<string, string>();
1162
+ private total = 0;
1163
+ private readonly budget: number;
1164
+
1165
+ constructor(budgetBytes: number = LABELS_PINS_BUDGET_BYTES) {
1166
+ this.budget = budgetBytes;
1167
+ }
1168
+
1169
+ get totalBytes(): number {
1170
+ return this.total;
1171
+ }
1172
+
1173
+ get budgetBytes(): number {
1174
+ return this.budget;
1175
+ }
1176
+
1177
+ setLabel(key: string, value: string): boolean {
1178
+ return this.setIn(this.labels, key, value);
1179
+ }
1180
+
1181
+ deleteLabel(key: string): void {
1182
+ const existing = this.labels.get(key);
1183
+ if (existing === undefined) return;
1184
+ this.total -= residentStringBytes(key) + residentStringBytes(existing) + RECORD_OBJECT_OVERHEAD_BYTES;
1185
+ this.labels.delete(key);
1186
+ }
1187
+
1188
+ setPin(key: string, value: string): boolean {
1189
+ return this.setIn(this.pins, key, value);
1190
+ }
1191
+
1192
+ getLabel(key: string): string | undefined {
1193
+ return this.labels.get(key);
1194
+ }
1195
+
1196
+ getPin(key: string): string | undefined {
1197
+ return this.pins.get(key);
1198
+ }
1199
+
1200
+ labelsEntries(): ReadonlyMap<string, string> {
1201
+ return this.labels;
1202
+ }
1203
+
1204
+ pinsEntries(): ReadonlyMap<string, string> {
1205
+ return this.pins;
1206
+ }
1207
+
1208
+ clear(): void {
1209
+ this.labels.clear();
1210
+ this.pins.clear();
1211
+ this.total = 0;
1212
+ }
1213
+
1214
+ private setIn(map: Map<string, string>, key: string, value: string): boolean {
1215
+ const cost = residentStringBytes(key) + residentStringBytes(value) + RECORD_OBJECT_OVERHEAD_BYTES;
1216
+ const existing = map.get(key);
1217
+ if (existing !== undefined) {
1218
+ const existingCost = residentStringBytes(key) + residentStringBytes(existing) + RECORD_OBJECT_OVERHEAD_BYTES;
1219
+ const delta = cost - existingCost;
1220
+ if (this.total + delta > this.budget) return false;
1221
+ this.total += delta;
1222
+ map.set(key, value);
1223
+ return true;
1224
+ }
1225
+ if (this.total + cost > this.budget) return false;
1226
+ this.total += cost;
1227
+ map.set(key, value);
1228
+ return true;
1229
+ }
1230
+ }
1231
+
1232
+ // ============================================================================
1233
+ // Persistent bounded parent→children artifact (disposable v2 parent index)
1234
+ // ============================================================================
1235
+
1236
+ /**
1237
+ * One parent→child edge plus the child's authoritative cold index fields. The
1238
+ * artifact is disposable derived proof; the `.spill.idx` remains authoritative
1239
+ * for entry offsets and the transcript remains authoritative for entry bytes.
1240
+ */
1241
+ export interface ParentBucketRecordInput {
1242
+ parentId: string;
1243
+ childId: string;
1244
+ ordinal: number;
1245
+ seq: number;
1246
+ byteOffset: number;
1247
+ byteLength: number;
1248
+ recordDigest: string;
1249
+ entryType?: string;
1250
+ }
1251
+
1252
+ /** Committed on-disk state of one parent bucket (exact bytes). */
1253
+ export interface ParentBucketCommit {
1254
+ size: number;
1255
+ digest: string;
1256
+ complete: boolean;
1257
+ }
1258
+
1259
+ /**
1260
+ * Commit-marker binding for the disposable parent artifact. The artifact
1261
+ * covers exactly the entry set of the `.spill.idx` whose digest is bound here;
1262
+ * a binding mismatch fails closed to the authoritative cold scan.
1263
+ */
1264
+ export interface ParentArtifactCommit {
1265
+ bucketCount: number;
1266
+ indexDigest: string;
1267
+ buckets: ParentBucketCommit[];
1268
+ }
1269
+
1270
+ /**
1271
+ * Deterministic hash-partition assignment for a parent id. Bucket files are
1272
+ * append-only journals; a lookup reads exactly one bucket (bounded range read)
1273
+ * and verifies each record's `parentId` to filter hash collisions.
1274
+ */
1275
+ export function parentBucketForId(parentId: string, bucketCount = PARENT_CHILDREN_BUCKET_COUNT): number {
1276
+ let hash = 0x811c9dc5;
1277
+ for (let index = 0; index < parentId.length; index++) {
1278
+ const code = parentId.charCodeAt(index);
1279
+ hash ^= code & 0xff;
1280
+ hash = Math.imul(hash, 0x01000193);
1281
+ hash ^= code >>> 8;
1282
+ hash = Math.imul(hash, 0x01000193);
1283
+ }
1284
+ return (hash >>> 0) % bucketCount;
1285
+ }
1286
+
1287
+ /** Serialize one parent→child edge as a compact JSON journal line (trailing newline). */
1288
+ export function serializeParentBucketRecord(input: ParentBucketRecordInput): string {
1289
+ return `${JSON.stringify({
1290
+ p: input.parentId,
1291
+ c: input.childId,
1292
+ o: input.ordinal,
1293
+ s: input.seq,
1294
+ b: input.byteOffset,
1295
+ l: input.byteLength,
1296
+ d: input.recordDigest,
1297
+ ...(input.entryType !== undefined ? { t: input.entryType } : {}),
1298
+ })}\n`;
1299
+ }
1300
+
1301
+ /** Strictly parse one parent bucket journal line; `undefined` marks the bucket corrupt. */
1302
+ export function parseParentBucketRecord(line: string): ParentBucketRecordInput | undefined {
1303
+ let value: Record<string, unknown>;
1304
+ try {
1305
+ value = JSON.parse(line) as Record<string, unknown>;
1306
+ } catch {
1307
+ return undefined;
1308
+ }
1309
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
1310
+ const { p, c, o, s, b, l, d, t } = value;
1311
+ if (typeof p !== "string" || p.length === 0) return undefined;
1312
+ if (typeof c !== "string" || c.length === 0) return undefined;
1313
+ if (typeof o !== "number" || typeof s !== "number" || typeof b !== "number" || typeof l !== "number")
1314
+ return undefined;
1315
+ if (!Number.isSafeInteger(o) || !Number.isSafeInteger(s) || !Number.isSafeInteger(b) || !Number.isSafeInteger(l))
1316
+ return undefined;
1317
+ if (typeof d !== "string" || d.length !== 64) return undefined;
1318
+ if (t !== undefined && typeof t !== "string") return undefined;
1319
+ return {
1320
+ parentId: p,
1321
+ childId: c,
1322
+ ordinal: o,
1323
+ seq: s,
1324
+ byteOffset: b,
1325
+ byteLength: l,
1326
+ recordDigest: d,
1327
+ ...(t !== undefined ? { entryType: t } : {}),
1328
+ };
1329
+ }
1330
+
1331
+ export interface ParentArtifactBuildResult {
1332
+ metadata: ParentArtifactCommit;
1333
+ /** Per-bucket records in physical (transcript) order. */
1334
+ buckets: string[][];
1335
+ /** Parents excluded because a bound (children/budget/parents) was exceeded. */
1336
+ excludedParents: readonly string[];
1337
+ }
1338
+
1339
+ /**
1340
+ * Bounded builder for the persistent parent artifact. Parents are indexed
1341
+ * fully or not at all: a parent whose children would exceed
1342
+ * `PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT`, whose first record would exceed
1343
+ * `PARENT_CHILDREN_BUDGET_BYTES`, or that would exceed
1344
+ * `PARENT_CHILDREN_MAX_PARENTS` is excluded (lookups for it fail closed), and
1345
+ * its buffered records are dropped so a truncated "complete" result can never
1346
+ * be published. Per-parent child order is preserved (insertion order).
1347
+ */
1348
+ export class BoundedParentArtifactBuilder {
1349
+ private readonly parents = new Map<string, { records: string[]; bytes: number }>();
1350
+ private readonly excluded = new Set<string>();
1351
+ private readonly incompleteBuckets = new Set<number>();
1352
+ private totalBytesValue = 0;
1353
+ private readonly maxParents: number;
1354
+ private readonly maxChildrenPerParent: number;
1355
+ private readonly budgetBytesValue: number;
1356
+ private readonly bucketCount: number;
1357
+ private finished = false;
1358
+ private capacityExhausted = false;
1359
+
1360
+ constructor(
1361
+ options: {
1362
+ maxParents?: number;
1363
+ maxChildrenPerParent?: number;
1364
+ budgetBytes?: number;
1365
+ bucketCount?: number;
1366
+ } = {},
1367
+ ) {
1368
+ this.maxParents = options.maxParents ?? PARENT_CHILDREN_MAX_PARENTS;
1369
+ this.maxChildrenPerParent = options.maxChildrenPerParent ?? PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT;
1370
+ this.budgetBytesValue = options.budgetBytes ?? PARENT_CHILDREN_BUDGET_BYTES;
1371
+ this.bucketCount = options.bucketCount ?? PARENT_CHILDREN_BUCKET_COUNT;
1372
+ }
1373
+
1374
+ get distinctParents(): number {
1375
+ return this.parents.size;
1376
+ }
1377
+
1378
+ get totalBytes(): number {
1379
+ return this.totalBytesValue;
1380
+ }
1381
+
1382
+ add(record: ParentBucketRecordInput): void {
1383
+ if (this.finished) throw new Error("parent_artifact_builder_finished");
1384
+ if (record.parentId.length === 0 || record.childId.length === 0) throw new Error("parent_artifact_empty_id");
1385
+ if (this.excluded.has(record.parentId)) return;
1386
+ if (this.capacityExhausted && !this.parents.has(record.parentId)) {
1387
+ this.incompleteBuckets.add(parentBucketForId(record.parentId, this.bucketCount));
1388
+ return;
1389
+ }
1390
+ const line = serializeParentBucketRecord(record);
1391
+ const bytes = Buffer.byteLength(line, "utf8");
1392
+ let parent = this.parents.get(record.parentId);
1393
+ if (parent === undefined) {
1394
+ if (this.parents.size >= this.maxParents || this.totalBytesValue + bytes > this.budgetBytesValue) {
1395
+ this.capacityExhausted = true;
1396
+ this.incompleteBuckets.add(parentBucketForId(record.parentId, this.bucketCount));
1397
+ return;
1398
+ }
1399
+ parent = { records: [], bytes: 0 };
1400
+ this.parents.set(record.parentId, parent);
1401
+ }
1402
+ if (parent.records.length >= this.maxChildrenPerParent || this.totalBytesValue + bytes > this.budgetBytesValue) {
1403
+ this.exclude(record.parentId);
1404
+ return;
1405
+ }
1406
+ parent.records.push(line);
1407
+ parent.bytes += bytes;
1408
+ this.totalBytesValue += bytes;
1409
+ }
1410
+
1411
+ private exclude(parentId: string): void {
1412
+ const parent = this.parents.get(parentId);
1413
+ if (parent) {
1414
+ this.totalBytesValue -= parent.bytes;
1415
+ this.parents.delete(parentId);
1416
+ }
1417
+ this.incompleteBuckets.add(parentBucketForId(parentId, this.bucketCount));
1418
+ this.excluded.add(parentId);
1419
+ }
1420
+
1421
+ finish(indexDigest: string): ParentArtifactBuildResult {
1422
+ if (this.finished) throw new Error("parent_artifact_builder_already_finished");
1423
+ this.finished = true;
1424
+ const byBucket: Array<{ records: string[]; bytes: number }> = Array.from({ length: this.bucketCount }, () => ({
1425
+ records: [],
1426
+ bytes: 0,
1427
+ }));
1428
+ for (const [parentId, parent] of this.parents) {
1429
+ const bucket = parentBucketForId(parentId, this.bucketCount);
1430
+ for (const line of parent.records) {
1431
+ byBucket[bucket].records.push(line);
1432
+ byBucket[bucket].bytes += Buffer.byteLength(line, "utf8");
1433
+ }
1434
+ }
1435
+ const buckets = byBucket.map((bucket, bucketIndex) => {
1436
+ const hash = createHash("sha256");
1437
+ for (const line of bucket.records) hash.update(line, "utf8");
1438
+ return {
1439
+ size: bucket.bytes,
1440
+ digest: hash.digest("hex"),
1441
+ complete: !this.incompleteBuckets.has(bucketIndex),
1442
+ };
1443
+ });
1444
+ const result: ParentArtifactBuildResult = {
1445
+ metadata: { bucketCount: this.bucketCount, indexDigest, buckets },
1446
+ buckets: byBucket.map(bucket => bucket.records),
1447
+ excludedParents: [...this.excluded],
1448
+ };
1449
+ this.parents.clear();
1450
+ this.excluded.clear();
1451
+ this.incompleteBuckets.clear();
1452
+ this.totalBytesValue = 0;
1453
+ return result;
1454
+ }
1455
+ }
1456
+
1457
+ /** Fixed in-memory cost of the retained parent-artifact runtime state (block-cache charged). */
1458
+ export function parentArtifactRuntimeBytes(bucketCount = PARENT_CHILDREN_BUCKET_COUNT): number {
1459
+ return bucketCount * 96 + 128;
1460
+ }
1461
+
1462
+ // ============================================================================
1463
+ // Persistent bounded dictionary artifact (deterministic hash partitions)
1464
+ // ============================================================================
1465
+
1466
+ /** Fixed partition count for the on-disk dictionary artifact (4 × 4 MiB buffers). */
1467
+ export const DICTIONARY_PARTITION_COUNT = 4;
1468
+ /** Maximum unique duplicate record ids retained as diagnostics in the meta. */
1469
+ export const DICTIONARY_DUPLICATE_DIAGNOSTIC_MAX = 64;
1470
+ /** Fixed capacity of the bounded duplicate-id detector (open addressing). */
1471
+ export const DICTIONARY_ID_SET_CAPACITY = 1_125_001;
1472
+ /** Maximum records the bounded duplicate-id detector can prove unique. */
1473
+ export const DICTIONARY_ID_SET_MAX_RECORDS = 1_000_000;
1474
+
1475
+ /**
1476
+ * One partition journal record. `term` is the lossless record id (never
1477
+ * truncated or hash-substituted); the remaining fields are the same exact
1478
+ * index fields the covered `.spill.idx` records, so a partition is a
1479
+ * self-sufficient hash-partitioned flat index for one id.
1480
+ */
1481
+ export interface DictionaryPartitionRecord {
1482
+ term: string;
1483
+ dictId: number;
1484
+ ordinal: number;
1485
+ seq: number;
1486
+ byteOffset: number;
1487
+ byteLength: number;
1488
+ recordDigest: string;
1489
+ parentId: string | null;
1490
+ entryType: string;
1491
+ }
1492
+
1493
+ /** Serialize one dictionary partition record as a compact JSON journal line (trailing newline). */
1494
+ export function serializeDictionaryPartitionRecord(input: DictionaryPartitionRecord): string {
1495
+ return `${JSON.stringify({
1496
+ t: input.term,
1497
+ i: input.dictId,
1498
+ o: input.ordinal,
1499
+ s: input.seq,
1500
+ b: input.byteOffset,
1501
+ l: input.byteLength,
1502
+ d: input.recordDigest,
1503
+ p: input.parentId,
1504
+ e: input.entryType,
1505
+ })}\n`;
1506
+ }
1507
+
1508
+ /** Strictly parse one dictionary partition journal line; `undefined` marks the partition corrupt. */
1509
+ export function parseDictionaryPartitionRecord(line: string): DictionaryPartitionRecord | undefined {
1510
+ let value: Record<string, unknown>;
1511
+ try {
1512
+ value = JSON.parse(line) as Record<string, unknown>;
1513
+ } catch {
1514
+ return undefined;
1515
+ }
1516
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
1517
+ const { t, i, o, s, b, l, d, p, e } = value;
1518
+ if (typeof t !== "string" || t.length === 0) return undefined;
1519
+ if (typeof i !== "number" || !Number.isSafeInteger(i) || i < 0) return undefined;
1520
+ if (
1521
+ typeof o !== "number" ||
1522
+ typeof s !== "number" ||
1523
+ typeof b !== "number" ||
1524
+ typeof l !== "number" ||
1525
+ !Number.isSafeInteger(o) ||
1526
+ !Number.isSafeInteger(s) ||
1527
+ !Number.isSafeInteger(b) ||
1528
+ !Number.isSafeInteger(l) ||
1529
+ o < 0 ||
1530
+ s < 0 ||
1531
+ b < 0 ||
1532
+ l <= 0
1533
+ )
1534
+ return undefined;
1535
+ if (typeof d !== "string" || !/^[0-9a-f]{64}$/.test(d)) return undefined;
1536
+ if (p !== null && typeof p !== "string") return undefined;
1537
+ if (typeof e !== "string" || e.length === 0) return undefined;
1538
+ return {
1539
+ term: t,
1540
+ dictId: i,
1541
+ ordinal: o,
1542
+ seq: s,
1543
+ byteOffset: b,
1544
+ byteLength: l,
1545
+ recordDigest: d,
1546
+ parentId: p,
1547
+ entryType: e,
1548
+ };
1549
+ }
1550
+
1551
+ /**
1552
+ * Deterministic hash-partition assignment for a record id (same FNV-1a family
1553
+ * as `parentBucketForId`). The same id always lands in the same partition, so
1554
+ * every occurrence of a duplicate id is confined to one journal.
1555
+ */
1556
+ export function dictionaryPartitionForId(id: string, partitionCount = DICTIONARY_PARTITION_COUNT): number {
1557
+ let hash = 0x811c9dc5;
1558
+ for (let index = 0; index < id.length; index++) {
1559
+ const code = id.charCodeAt(index);
1560
+ hash ^= code & 0xff;
1561
+ hash = Math.imul(hash, 0x01000193);
1562
+ hash ^= code >>> 8;
1563
+ hash = Math.imul(hash, 0x01000193);
1564
+ }
1565
+ return (hash >>> 0) % partitionCount;
1566
+ }
1567
+
1568
+ /** Committed exact bytes of one on-disk dictionary partition. */
1569
+ export interface DictionaryPartitionCommit {
1570
+ size: number;
1571
+ digest: string;
1572
+ records: number;
1573
+ complete: boolean;
1574
+ }
1575
+
1576
+ /**
1577
+ * Commit-marker binding for the disposable dictionary artifact. Every partition
1578
+ * and the meta file are bound by exact size + SHA-256, and the artifact covers
1579
+ * exactly the entry set of the `.spill.idx` whose digest is bound here; a
1580
+ * binding mismatch fails closed to the authoritative cold scan.
1581
+ */
1582
+ export interface DictionaryArtifactCommit {
1583
+ header: SidecarIndexHeaderV2;
1584
+ /** Exact `.spill.idx` digest the artifact covers. */
1585
+ indexDigest: string;
1586
+ /** Per-partition exact-byte commits, one per partition file. */
1587
+ partitions: DictionaryPartitionCommit[];
1588
+ /** Exact bytes of the `.spill.dict-meta` file (self-referential fields omitted from the payload). */
1589
+ metaSize: number;
1590
+ metaDigest: string;
1591
+ recordCount: number;
1592
+ uniqueTerms: number;
1593
+ /** Combined serialized term bytes (lossless ids). */
1594
+ totalBytes: number;
1595
+ /** Bounded duplicate-id diagnostics; non-empty ⇒ the artifact must never be adopted. */
1596
+ duplicateIds: readonly string[];
1597
+ sidecarIneligible: boolean;
1598
+ }
1599
+
1600
+ /**
1601
+ * Exact serialized bytes of the meta file. `metaSize`/`metaDigest` are omitted
1602
+ * from the payload (they authenticate the bytes that omit them), so
1603
+ * serialization is deterministic and self-referentiality is impossible.
1604
+ */
1605
+ export function dictionaryArtifactMetaBytes(commit: Omit<DictionaryArtifactCommit, "metaSize" | "metaDigest">): string {
1606
+ return `${JSON.stringify({
1607
+ header: {
1608
+ version: commit.header.version,
1609
+ sessionId: commit.header.sessionId,
1610
+ sidecarIneligible: commit.header.sidecarIneligible,
1611
+ },
1612
+ indexDigest: commit.indexDigest,
1613
+ partitions: commit.partitions.map(partition => ({
1614
+ size: partition.size,
1615
+ digest: partition.digest,
1616
+ records: partition.records,
1617
+ complete: partition.complete,
1618
+ })),
1619
+ recordCount: commit.recordCount,
1620
+ uniqueTerms: commit.uniqueTerms,
1621
+ totalBytes: commit.totalBytes,
1622
+ duplicateIds: [...commit.duplicateIds],
1623
+ sidecarIneligible: commit.sidecarIneligible,
1624
+ })}\n`;
1625
+ }
1626
+
1627
+ /** Compute the exact meta bytes and the resulting self-authenticating commit. */
1628
+ export function finalizeDictionaryArtifactCommit(commit: Omit<DictionaryArtifactCommit, "metaSize" | "metaDigest">): {
1629
+ bytes: string;
1630
+ commit: DictionaryArtifactCommit;
1631
+ } {
1632
+ const bytes = dictionaryArtifactMetaBytes(commit);
1633
+ const metaDigest = computeLineDigest(Buffer.from(bytes, "utf8"));
1634
+ return { bytes, commit: { ...commit, metaSize: bytes.length, metaDigest } };
1635
+ }
1636
+
1637
+ /**
1638
+ * Strictly parse the exact meta bytes. The digest and size are recomputed over
1639
+ * the supplied exact bytes; any structural, bound, or digest mismatch returns
1640
+ * `undefined` (the artifact is then never trusted).
1641
+ */
1642
+ export function parseDictionaryArtifactCommit(bytes: string | Uint8Array): DictionaryArtifactCommit | undefined {
1643
+ const text = typeof bytes === "string" ? bytes : new TextDecoder("utf-8").decode(bytes);
1644
+ const exactLength = typeof bytes === "string" ? Buffer.byteLength(bytes, "utf8") : bytes.byteLength;
1645
+ let value: unknown;
1646
+ try {
1647
+ value = JSON.parse(text);
1648
+ } catch {
1649
+ return undefined;
1650
+ }
1651
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
1652
+ const record = value as Record<string, unknown>;
1653
+ const header = record.header;
1654
+ if (header === null || typeof header !== "object" || Array.isArray(header)) return undefined;
1655
+ const headerRecord = header as Record<string, unknown>;
1656
+ if (headerRecord.version !== 2) return undefined;
1657
+ if (typeof headerRecord.sessionId !== "string") return undefined;
1658
+ if (typeof headerRecord.sidecarIneligible !== "boolean") return undefined;
1659
+ if (typeof record.indexDigest !== "string" || !/^[0-9a-f]{64}$/.test(record.indexDigest)) return undefined;
1660
+ if (!Array.isArray(record.partitions) || record.partitions.length !== DICTIONARY_PARTITION_COUNT) return undefined;
1661
+ const partitions: DictionaryPartitionCommit[] = [];
1662
+ for (const candidate of record.partitions) {
1663
+ if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) return undefined;
1664
+ const partition = candidate as Record<string, unknown>;
1665
+ if (!Number.isSafeInteger(partition.size) || (partition.size as number) < 0) return undefined;
1666
+ if (typeof partition.digest !== "string" || !/^[0-9a-f]{64}$/.test(partition.digest)) return undefined;
1667
+ if (!Number.isSafeInteger(partition.records) || (partition.records as number) < 0) return undefined;
1668
+ if (typeof partition.complete !== "boolean") return undefined;
1669
+ partitions.push({
1670
+ size: partition.size as number,
1671
+ digest: partition.digest,
1672
+ records: partition.records as number,
1673
+ complete: partition.complete,
1674
+ });
1675
+ }
1676
+ if (!Number.isSafeInteger(record.recordCount) || (record.recordCount as number) < 0) return undefined;
1677
+ if (!Number.isSafeInteger(record.uniqueTerms) || (record.uniqueTerms as number) < 0) return undefined;
1678
+ if (!Number.isSafeInteger(record.totalBytes) || (record.totalBytes as number) < 0) return undefined;
1679
+ if (!Array.isArray(record.duplicateIds) || record.duplicateIds.some(candidate => typeof candidate !== "string"))
1680
+ return undefined;
1681
+ if (typeof record.sidecarIneligible !== "boolean") return undefined;
1682
+ const expectedDigest = computeLineDigest(
1683
+ typeof bytes === "string" ? Buffer.from(bytes, "utf8") : Buffer.from(bytes),
1684
+ );
1685
+ return {
1686
+ header: {
1687
+ version: 2,
1688
+ sessionId: headerRecord.sessionId,
1689
+ sidecarIneligible: headerRecord.sidecarIneligible,
1690
+ },
1691
+ indexDigest: record.indexDigest,
1692
+ partitions,
1693
+ metaSize: exactLength,
1694
+ metaDigest: expectedDigest,
1695
+ recordCount: record.recordCount as number,
1696
+ uniqueTerms: record.uniqueTerms as number,
1697
+ totalBytes: record.totalBytes as number,
1698
+ duplicateIds: record.duplicateIds as string[],
1699
+ sidecarIneligible: record.sidecarIneligible,
1700
+ };
1701
+ }
1702
+
1703
+ /** Bounded duplicate-id oracle for the streaming dictionary build. */
1704
+ export interface DictionaryIdDetector {
1705
+ add(id: string): "added" | "duplicate" | "full";
1706
+ }
1707
+
1708
+ /**
1709
+ * Bounded duplicate-id oracle. Small transcripts stay in a tiny exact set;
1710
+ * larger transcripts promote once to the fixed-capacity 48-bit hash table.
1711
+ * A hash collision can only produce a spurious duplicate (fail closed), never
1712
+ * miss a true duplicate.
1713
+ */
1714
+ export class BoundedDictionaryIdSet implements DictionaryIdDetector {
1715
+ #small: Set<string> | undefined = new Set();
1716
+ #high: Uint16Array | undefined;
1717
+ #low: Uint32Array | undefined;
1718
+ #size = 0;
1719
+
1720
+ #hash(value: string, seed: number): number {
1721
+ let hash = seed >>> 0;
1722
+ for (let index = 0; index < value.length; index++) {
1723
+ hash ^= value.charCodeAt(index);
1724
+ hash = Math.imul(hash, 0x01000193) >>> 0;
1725
+ }
1726
+ return hash;
1727
+ }
1728
+
1729
+ #hashPair(value: string): [number, number] {
1730
+ let high = this.#hash(value, 0x811c9dc5) & 0xffff;
1731
+ const low = this.#hash(value, 0x9e3779b9);
1732
+ if (high === 0 && low === 0) high = 1;
1733
+ return [high, low];
1734
+ }
1735
+
1736
+ #insertHash(high: number, low: number): "added" | "duplicate" | "full" {
1737
+ const highs = this.#high!;
1738
+ const lows = this.#low!;
1739
+ let slot = low % DICTIONARY_ID_SET_CAPACITY;
1740
+ for (let probes = 0; probes < DICTIONARY_ID_SET_CAPACITY; probes++) {
1741
+ const existingHigh = highs[slot];
1742
+ const existingLow = lows[slot];
1743
+ if (existingHigh === high && existingLow === low) return "duplicate";
1744
+ if (existingHigh === 0 && existingLow === 0) {
1745
+ highs[slot] = high;
1746
+ lows[slot] = low;
1747
+ return "added";
1748
+ }
1749
+ slot++;
1750
+ if (slot === DICTIONARY_ID_SET_CAPACITY) slot = 0;
1751
+ }
1752
+ return "full";
1753
+ }
1754
+
1755
+ #promote(): void {
1756
+ this.#high = new Uint16Array(DICTIONARY_ID_SET_CAPACITY);
1757
+ this.#low = new Uint32Array(DICTIONARY_ID_SET_CAPACITY);
1758
+ for (const value of this.#small!) {
1759
+ const [high, low] = this.#hashPair(value);
1760
+ this.#insertHash(high, low);
1761
+ }
1762
+ this.#small = undefined;
1763
+ }
1764
+
1765
+ has(value: string): boolean {
1766
+ if (this.#small) return this.#small.has(value);
1767
+ const [high, low] = this.#hashPair(value);
1768
+ const highs = this.#high!;
1769
+ const lows = this.#low!;
1770
+ let slot = low % DICTIONARY_ID_SET_CAPACITY;
1771
+ for (let probes = 0; probes < DICTIONARY_ID_SET_CAPACITY; probes++) {
1772
+ const existingHigh = highs[slot];
1773
+ const existingLow = lows[slot];
1774
+ if (existingHigh === 0 && existingLow === 0) return false;
1775
+ if (existingHigh === high && existingLow === low) return true;
1776
+ slot++;
1777
+ if (slot === DICTIONARY_ID_SET_CAPACITY) slot = 0;
1778
+ }
1779
+ return true;
1780
+ }
1781
+
1782
+ add(value: string): "added" | "duplicate" | "full" {
1783
+ if (this.#size >= DICTIONARY_ID_SET_MAX_RECORDS) return "full";
1784
+ if (this.#small) {
1785
+ if (this.#small.has(value)) return "duplicate";
1786
+ if (this.#small.size < 4096) {
1787
+ this.#small.add(value);
1788
+ this.#size++;
1789
+ return "added";
1790
+ }
1791
+ this.#promote();
1792
+ }
1793
+ const [high, low] = this.#hashPair(value);
1794
+ const result = this.#insertHash(high, low);
1795
+ if (result === "added") this.#size++;
1796
+ return result;
1797
+ }
1798
+ }
1799
+
1800
+ /** One record consumed by the streaming dictionary artifact builder. */
1801
+ export interface DictionaryArtifactRecordInput {
1802
+ id: string;
1803
+ ordinal: number;
1804
+ seq: number;
1805
+ byteOffset: number;
1806
+ byteLength: number;
1807
+ recordDigest: string;
1808
+ parentId: string | null;
1809
+ entryType: string;
1810
+ }
1811
+
1812
+ /**
1813
+ * Persistence target for flushed partition lines. The target owns the exact
1814
+ * bytes on disk and their running digests (it must keep per-partition hashes
1815
+ * for append-time rebinding); the builder never retains flushed lines.
1816
+ */
1817
+ export interface DictionaryArtifactFlushTarget {
1818
+ /** Append `lines` to `partition`'s journal, fsyncing before returning false on failure. */
1819
+ writePartitionLines(partition: number, lines: readonly string[]): boolean;
1820
+ /** Exact-byte commit of one partition as currently persisted. */
1821
+ getPartitionCommit(partition: number): DictionaryPartitionCommit;
1822
+ }
1823
+
1824
+ export type DictionaryArtifactAddResult =
1825
+ | { kind: "ok" }
1826
+ | { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number }
1827
+ | { kind: "flush_failed" };
1828
+
1829
+ export type DictionaryArtifactBuildResult =
1830
+ | { kind: "ok"; commit: DictionaryArtifactCommit; stats: DictionaryBuildStats }
1831
+ | { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number }
1832
+ | { kind: "flush_failed" };
1833
+
1834
+ export interface DictionaryArtifactBuildOptions {
1835
+ peakBudgetBytes?: number;
1836
+ partitionBufferBytes?: number;
1837
+ journalBytes?: number;
1838
+ partitionCount?: number;
1839
+ detector?: DictionaryIdDetector;
1840
+ /** Persistence target for flushed partition lines; required for `add`/`finish`. */
1841
+ target?: DictionaryArtifactFlushTarget;
1842
+ }
1843
+
1844
+ /**
1845
+ * Streaming dictionary artifact builder with a deterministic 20 MiB build peak
1846
+ * (4 × `partitionBufferBytes` partition buffers + one `journalBytes` flush
1847
+ * buffer). Consumes records one at a time, flushes full partition buffers
1848
+ * through the journal to the flush target, and never retains the complete
1849
+ * dictionary object/JSON graph. Record ids are stored losslessly as terms.
1850
+ * Duplicate record ids mark the artifact `sidecarIneligible` (never adopted;
1851
+ * the session falls back to eager) while the build still completes
1852
+ * deterministically.
1853
+ */
1854
+ export class BoundedDictionaryArtifactBuilder {
1855
+ private readonly buffers: string[][] = [];
1856
+ private readonly bufferBytes: number[] = [];
1857
+ private journal: string[] = [];
1858
+ private journalBytes = 0;
1859
+ private journalPartition: number | undefined;
1860
+ private readonly partitionCountValue: number;
1861
+ private readonly partitionBufferBytesValue: number;
1862
+ private readonly journalBytesValue: number;
1863
+ private readonly peakBudgetValue: number;
1864
+ private readonly detector: DictionaryIdDetector;
1865
+ private readonly target: DictionaryArtifactFlushTarget | undefined;
1866
+ private readonly duplicateIds: string[] = [];
1867
+ private duplicateIdBytes = 0;
1868
+ private uniqueTerms = 0;
1869
+ private totalTermBytes = 0;
1870
+ private recordCount = 0;
1871
+ private peakBytesValue = 0;
1872
+ private sidecarIneligible = false;
1873
+ private flushFailed = false;
1874
+ private finished = false;
1875
+
1876
+ constructor(options: DictionaryArtifactBuildOptions = {}) {
1877
+ this.peakBudgetValue = options.peakBudgetBytes ?? DICTIONARY_BUILD_PEAK_BYTES;
1878
+ this.partitionCountValue = options.partitionCount ?? DICTIONARY_PARTITION_COUNT;
1879
+ this.partitionBufferBytesValue = options.partitionBufferBytes ?? DICTIONARY_PARTITION_BUFFER_BYTES;
1880
+ this.journalBytesValue = options.journalBytes ?? DICTIONARY_BUCKET_JOURNAL_BYTES;
1881
+ this.detector = options.detector ?? new BoundedDictionaryIdSet();
1882
+ this.target = options.target;
1883
+ for (let partition = 0; partition < this.partitionCountValue; partition++) {
1884
+ this.buffers.push([]);
1885
+ this.bufferBytes.push(0);
1886
+ }
1887
+ }
1888
+
1889
+ get peakBytes(): number {
1890
+ return this.peakBytesValue;
1891
+ }
1892
+
1893
+ /** Consumes one record. The build peak stays within the 20 MiB buffer model. */
1894
+ add(record: DictionaryArtifactRecordInput): DictionaryArtifactAddResult {
1895
+ if (this.finished) throw new Error("dictionary_artifact_builder_finished");
1896
+ this.recordCount += 1;
1897
+ const partition = dictionaryPartitionForId(record.id, this.partitionCountValue);
1898
+ const detection = this.detector.add(record.id);
1899
+ if (detection === "duplicate") {
1900
+ this.sidecarIneligible = true;
1901
+ if (this.duplicateIds.length < DICTIONARY_DUPLICATE_DIAGNOSTIC_MAX && !this.duplicateIds.includes(record.id)) {
1902
+ this.duplicateIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
1903
+ this.duplicateIds.push(record.id);
1904
+ }
1905
+ } else if (detection === "full") {
1906
+ // Uniqueness can no longer be proven with bounded memory; fail closed.
1907
+ this.sidecarIneligible = true;
1908
+ }
1909
+ if (detection === "added") {
1910
+ this.uniqueTerms += 1;
1911
+ this.totalTermBytes += Buffer.byteLength(record.id, "utf8");
1912
+ }
1913
+ const line = serializeDictionaryPartitionRecord({
1914
+ term: record.id,
1915
+ dictId: detection === "added" ? this.uniqueTerms - 1 : -1,
1916
+ ordinal: record.ordinal,
1917
+ seq: record.seq,
1918
+ byteOffset: record.byteOffset,
1919
+ byteLength: record.byteLength,
1920
+ recordDigest: record.recordDigest,
1921
+ parentId: record.parentId,
1922
+ entryType: record.entryType,
1923
+ });
1924
+ const lineBytes = Buffer.byteLength(line, "utf8");
1925
+ this.buffers[partition]!.push(line);
1926
+ this.bufferBytes[partition]! += lineBytes;
1927
+ this.updatePeak();
1928
+ if (this.peakBytesValue > this.peakBudgetValue) {
1929
+ return { kind: "budget_exceeded", peakBytes: this.peakBytesValue, budgetBytes: this.peakBudgetValue };
1930
+ }
1931
+ if (this.bufferBytes[partition]! > this.partitionBufferBytesValue) {
1932
+ if (!this.flushPartition(partition)) return { kind: "flush_failed" };
1933
+ }
1934
+ return { kind: "ok" };
1935
+ }
1936
+
1937
+ /** Flush every buffer, read the exact partition commits from the target, and finalize. */
1938
+ finish(sessionId: string, indexDigest: string): DictionaryArtifactBuildResult {
1939
+ if (this.finished) throw new Error("dictionary_artifact_builder_already_finished");
1940
+ this.finished = true;
1941
+ for (let partition = 0; partition < this.partitionCountValue; partition++) {
1942
+ if (this.bufferBytes[partition]! > 0) {
1943
+ if (!this.flushPartition(partition)) return { kind: "flush_failed" };
1944
+ }
1945
+ }
1946
+ if (this.flushFailed) return { kind: "flush_failed" };
1947
+ if (this.peakBytesValue > this.peakBudgetValue) {
1948
+ return { kind: "budget_exceeded", peakBytes: this.peakBytesValue, budgetBytes: this.peakBudgetValue };
1949
+ }
1950
+ if (!this.target) return { kind: "flush_failed" };
1951
+ const partitions: DictionaryPartitionCommit[] = [];
1952
+ for (let partition = 0; partition < this.partitionCountValue; partition++) {
1953
+ partitions.push(this.target.getPartitionCommit(partition));
1954
+ }
1955
+ const commit = {
1956
+ header: { version: 2 as const, sessionId, sidecarIneligible: this.sidecarIneligible },
1957
+ indexDigest,
1958
+ partitions,
1959
+ metaSize: 0,
1960
+ metaDigest: "",
1961
+ recordCount: this.recordCount,
1962
+ uniqueTerms: this.uniqueTerms,
1963
+ totalBytes: this.totalTermBytes,
1964
+ duplicateIds: [...this.duplicateIds],
1965
+ sidecarIneligible: this.sidecarIneligible,
1966
+ };
1967
+ const stats: DictionaryBuildStats = {
1968
+ totalRecords: this.recordCount,
1969
+ uniqueTerms: this.uniqueTerms,
1970
+ totalBytes: this.totalTermBytes,
1971
+ peakBytes: this.peakBytesValue,
1972
+ duplicateIds: [...this.duplicateIds],
1973
+ sidecarIneligible: this.sidecarIneligible,
1974
+ };
1975
+ this.journal = [];
1976
+ this.journalBytes = 0;
1977
+ this.journalPartition = undefined;
1978
+ for (let partition = 0; partition < this.partitionCountValue; partition++) {
1979
+ this.buffers[partition] = [];
1980
+ this.bufferBytes[partition] = 0;
1981
+ }
1982
+ return { kind: "ok", commit, stats };
1983
+ }
1984
+
1985
+ /** Moves one partition's buffer into the journal (≤ 4 MiB) and flushes it to the target. */
1986
+ private flushPartition(partition: number): boolean {
1987
+ const buffer = this.buffers[partition]!;
1988
+ if (buffer.length === 0) return true;
1989
+ if (this.journalPartition !== undefined) {
1990
+ // The previous journal flush failed or was never drained; fail closed.
1991
+ this.flushFailed = true;
1992
+ return false;
1993
+ }
1994
+ // Take as many leading lines as fit the journal cap (≤ 4 MiB); the rest
1995
+ // stays buffered so a flush batch never exceeds the fixed journal bound.
1996
+ let take = 0;
1997
+ let takeBytes = 0;
1998
+ while (take < buffer.length) {
1999
+ const lineBytes = Buffer.byteLength(buffer[take]!, "utf8");
2000
+ if (takeBytes + lineBytes > this.journalBytesValue) break;
2001
+ takeBytes += lineBytes;
2002
+ take++;
2003
+ }
2004
+ if (take === 0) {
2005
+ // A single line exceeds the journal cap; fail closed.
2006
+ this.flushFailed = true;
2007
+ return false;
2008
+ }
2009
+ const lines = take === buffer.length ? buffer : buffer.slice(0, take);
2010
+ this.buffers[partition] = take === buffer.length ? [] : buffer.slice(take);
2011
+ this.bufferBytes[partition]! -= takeBytes;
2012
+ this.journal = lines;
2013
+ this.journalBytes = takeBytes;
2014
+ this.journalPartition = partition;
2015
+ this.updatePeak();
2016
+ const ok = this.flushJournal();
2017
+ this.updatePeak();
2018
+ return ok;
2019
+ }
2020
+
2021
+ private flushJournal(): boolean {
2022
+ if (this.journal.length === 0) return true;
2023
+ const partition = this.journalPartition;
2024
+ if (partition === undefined) return true;
2025
+ const lines = this.journal;
2026
+ const bytes = this.journalBytes;
2027
+ this.journal = [];
2028
+ this.journalBytes = 0;
2029
+ this.journalPartition = undefined;
2030
+ const target = this.target;
2031
+ if (!target) {
2032
+ this.flushFailed = true;
2033
+ return false;
2034
+ }
2035
+ const ok = target.writePartitionLines(partition, lines);
2036
+ if (!ok) this.flushFailed = true;
2037
+ if (bytes > this.peakBytesValue) this.peakBytesValue = bytes;
2038
+ return ok;
2039
+ }
2040
+
2041
+ private updatePeak(): void {
2042
+ // Actual buffered content only: at most one partition buffer is being
2043
+ // filled (~4 MiB), the others hold flush remainders, and the journal is
2044
+ // capped at its fixed size — so the peak never exceeds the 20 MiB model
2045
+ // (4 × 4 MiB partition buffers + 4 MiB bucket flush).
2046
+ const current =
2047
+ this.bufferBytes.reduce((total, bytes) => total + bytes, 0) + this.journalBytes + this.duplicateIdBytes;
2048
+ if (current > this.peakBytesValue) this.peakBytesValue = current;
2049
+ }
2050
+ }
2051
+
2052
+ /** Fixed in-memory cost of the retained dictionary-artifact runtime state (block-cache charged). */
2053
+ export function dictionaryArtifactRuntimeBytes(partitionCount = DICTIONARY_PARTITION_COUNT): number {
2054
+ return partitionCount * 96 + 128;
2055
+ }
2056
+
2057
+ // ============================================================================
2058
+ // Persistent metadata-delta artifact (demoted reducer/provider values)
2059
+ // ============================================================================
2060
+
2061
+ /**
2062
+ * One demoted provider-state value descriptor: exact ordinal, kind, byte
2063
+ * location, and digest inside the `.spill.metadata-delta` section. The value
2064
+ * bytes are persisted before demotion, so a descriptor always points at
2065
+ * durable authenticated bytes.
2066
+ */
2067
+ export interface MetadataDeltaValue {
2068
+ /** Provider-state key (`providerStateEntryKey` semantics: type or `type:role`). */
2069
+ key: string;
2070
+ /** Entry type of the demoted value. */
2071
+ kind: string;
2072
+ /** Exact transcript ordinal of the demoted entry. */
2073
+ ordinal: number;
2074
+ /** Position in the build's ordered provider list (reopen merge slot). */
2075
+ position: number;
2076
+ offset: number;
2077
+ length: number;
2078
+ sha256: string;
2079
+ }
2080
+
2081
+ /** Commit-marker binding for the disposable metadata-delta section. */
2082
+ export interface MetadataDeltaArtifactCommit {
2083
+ /** Exact `.spill.idx` digest the sidecar set was built with. */
2084
+ indexDigest: string;
2085
+ /** Exact bytes of the `.spill.metadata-delta` file. */
2086
+ size: number;
2087
+ sha256: string;
2088
+ values: readonly MetadataDeltaValue[];
2089
+ }
2090
+
2091
+ /** Strict descriptor validation (any ambiguity fails closed). */
2092
+ export function isValidMetadataDeltaValue(value: unknown): value is MetadataDeltaValue {
2093
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
2094
+ const record = value as Record<string, unknown>;
2095
+ return (
2096
+ typeof record.key === "string" &&
2097
+ record.key.length > 0 &&
2098
+ typeof record.kind === "string" &&
2099
+ record.kind.length > 0 &&
2100
+ Number.isSafeInteger(record.ordinal) &&
2101
+ (record.ordinal as number) >= 0 &&
2102
+ Number.isSafeInteger(record.position) &&
2103
+ (record.position as number) >= 0 &&
2104
+ Number.isSafeInteger(record.offset) &&
2105
+ (record.offset as number) >= 0 &&
2106
+ Number.isSafeInteger(record.length) &&
2107
+ (record.length as number) > 0 &&
2108
+ typeof record.sha256 === "string" &&
2109
+ /^[0-9a-f]{64}$/.test(record.sha256)
2110
+ );
2111
+ }
2112
+
2113
+ /** Strict commit-binding validation: bounded values, unique keys/positions, in-range offsets. */
2114
+ export function isValidMetadataDeltaCommit(value: unknown): value is MetadataDeltaArtifactCommit {
2115
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
2116
+ const record = value as Record<string, unknown>;
2117
+ if (typeof record.indexDigest !== "string" || !/^[0-9a-f]{64}$/.test(record.indexDigest)) return false;
2118
+ if (!Number.isSafeInteger(record.size) || (record.size as number) < 0) return false;
2119
+ if (typeof record.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(record.sha256)) return false;
2120
+ if (!Array.isArray(record.values) || record.values.length > 256) return false;
2121
+ if (!record.values.every(isValidMetadataDeltaValue)) return false;
2122
+ const keys = new Set<string>();
2123
+ const positions = new Set<number>();
2124
+ for (const value of record.values) {
2125
+ // Each value is independently authenticated by its exact byte digest; the
2126
+ // marker may legitimately reorder re-demoted values relative to their
2127
+ // byte offsets, so only uniqueness and range bounds are enforced here.
2128
+ if (keys.has(value.key)) return false;
2129
+ if (positions.has(value.position)) return false;
2130
+ keys.add(value.key);
2131
+ positions.add(value.position);
2132
+ if (value.offset + value.length > (record.size as number)) return false;
2133
+ }
2134
+ return true;
2135
+ }
2136
+
2137
+ /** Compute the descriptor for one value whose exact line bytes are about to be persisted. */
2138
+ export function metadataDeltaValueDescriptor(input: {
2139
+ key: string;
2140
+ kind: string;
2141
+ ordinal: number;
2142
+ position: number;
2143
+ offset: number;
2144
+ lineBytes: Uint8Array;
2145
+ }): MetadataDeltaValue {
2146
+ return {
2147
+ key: input.key,
2148
+ kind: input.kind,
2149
+ ordinal: input.ordinal,
2150
+ position: input.position,
2151
+ offset: input.offset,
2152
+ length: input.lineBytes.byteLength,
2153
+ sha256: computeLineDigest(input.lineBytes),
2154
+ };
2155
+ }
2156
+
2157
+ /** Resident bytes of one retained metadata-delta descriptor (key + 24 B descriptor + object). */
2158
+ export function metadataDeltaDescriptorResidentBytes(key: string): number {
2159
+ return residentStringBytes(key) + DESCRIPTOR_BYTES + RECORD_OBJECT_OVERHEAD_BYTES;
2160
+ }