@gajae-code/coding-agent 0.12.18 → 0.12.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -9
- package/dist/types/commands/gc.d.ts +4 -0
- package/dist/types/commands/sdk.d.ts +66 -0
- package/dist/types/config/settings-schema.d.ts +34 -0
- package/dist/types/gjc-runtime/gc-runtime.d.ts +97 -1
- package/dist/types/modes/acp/acp-agent.d.ts +33 -1
- package/dist/types/modes/components/bash-execution.d.ts +7 -0
- package/dist/types/modes/components/eval-execution.d.ts +7 -0
- package/dist/types/modes/utils/ui-helpers.d.ts +10 -1
- package/dist/types/sdk/acp/adapter.d.ts +9 -17
- package/dist/types/sdk/acp/index.d.ts +1 -1
- package/dist/types/sdk/broker/broker.d.ts +93 -1
- package/dist/types/sdk/broker/ensure.d.ts +17 -0
- package/dist/types/sdk/broker/index.d.ts +1 -0
- package/dist/types/sdk/broker/lifecycle-ledger.d.ts +10 -1
- package/dist/types/sdk/broker/lifecycle.d.ts +81 -2
- package/dist/types/sdk/broker/startup-budget.d.ts +44 -0
- package/dist/types/sdk/bus/chat-daemon-control.d.ts +7 -1
- package/dist/types/sdk/bus/chat-daemon-runtime.d.ts +11 -1
- package/dist/types/sdk/bus/discord-daemon.d.ts +1 -0
- package/dist/types/sdk/bus/index.d.ts +20 -9
- package/dist/types/sdk/bus/prompt-reconciliation.d.ts +2 -6
- package/dist/types/sdk/bus/slack-daemon.d.ts +2 -2
- package/dist/types/sdk/bus/telegram-daemon-contract.d.ts +4 -2
- package/dist/types/sdk/host/query/revision-store.d.ts +5 -0
- package/dist/types/sdk/host/session-runtime.d.ts +4 -0
- package/dist/types/sdk/lifecycle-session.d.ts +13 -2
- package/dist/types/sdk/prompt-failure.d.ts +13 -0
- package/dist/types/sdk/prompt-watchdog.d.ts +82 -0
- package/dist/types/sdk/session-reconnect.d.ts +20 -0
- package/dist/types/sdk/startup-capability.d.ts +1 -1
- package/dist/types/session/agent-session.d.ts +14 -4
- package/dist/types/session/blob-store.d.ts +32 -0
- package/dist/types/session/internal/managed-session-storage.d.ts +40 -2
- package/dist/types/session/internal/session-memory-sidecar.d.ts +881 -0
- package/dist/types/session/session-manager.d.ts +301 -27
- package/dist/types/session/session-storage.d.ts +170 -0
- package/dist/types/task/executor.d.ts +2 -1
- package/dist/types/tools/bash.d.ts +1 -0
- package/dist/types/tui/output-block.d.ts +6 -0
- package/package.json +8 -8
- package/src/cli/args.ts +75 -34
- package/src/commands/acp.ts +7 -1
- package/src/commands/gc.ts +31 -2
- package/src/commands/sdk.ts +141 -7
- package/src/config/settings-schema.ts +43 -0
- package/src/coordinator-mcp/server.ts +6 -1
- package/src/export/html/index.ts +83 -39
- package/src/gjc-runtime/gc-runtime.ts +1283 -4
- package/src/internal-urls/docs-index.generated.ts +1 -1
- package/src/internal-urls/local-protocol.ts +14 -0
- package/src/main.ts +40 -9
- package/src/modes/acp/acp-agent.ts +564 -37
- package/src/modes/acp/terminal-auth.ts +2 -2
- package/src/modes/components/bash-execution.ts +14 -0
- package/src/modes/components/eval-execution.ts +14 -0
- package/src/modes/controllers/command-controller.ts +63 -8
- package/src/modes/utils/ui-helpers.ts +81 -11
- package/src/sdk/acp/adapter.ts +46 -45
- package/src/sdk/acp/index.ts +1 -1
- package/src/sdk/broker/broker.ts +392 -11
- package/src/sdk/broker/ensure.ts +131 -61
- package/src/sdk/broker/index.ts +1 -0
- package/src/sdk/broker/lifecycle-ledger.ts +59 -5
- package/src/sdk/broker/lifecycle.ts +278 -63
- package/src/sdk/broker/startup-budget.ts +103 -0
- package/src/sdk/bus/chat-daemon-control.ts +9 -3
- package/src/sdk/bus/chat-daemon-runtime.ts +565 -29
- package/src/sdk/bus/conversation-store.ts +1 -1
- package/src/sdk/bus/discord-daemon.ts +2 -1
- package/src/sdk/bus/discord-live-provider.ts +34 -6
- package/src/sdk/bus/index.ts +118 -20
- package/src/sdk/bus/kind-aware-reconciliation.ts +2 -1
- package/src/sdk/bus/prompt-reconciliation.ts +4 -9
- package/src/sdk/bus/slack-daemon.ts +91 -18
- package/src/sdk/bus/telegram-daemon-contract.ts +4 -2
- package/src/sdk/bus/topic-registry.ts +7 -1
- package/src/sdk/cli/session-cli.ts +8 -1
- package/src/sdk/host/query/handlers.ts +1 -1
- package/src/sdk/host/query/revision-store.ts +48 -12
- package/src/sdk/host/session-runtime.test.ts +50 -0
- package/src/sdk/host/session-runtime.ts +12 -2
- package/src/sdk/lifecycle-session.ts +26 -3
- package/src/sdk/mcp/server.ts +6 -1
- package/src/sdk/prompt-failure.ts +42 -0
- package/src/sdk/prompt-watchdog.ts +162 -0
- package/src/sdk/session-reconnect.ts +43 -0
- package/src/sdk/startup-capability.ts +13 -4
- package/src/session/agent-session.ts +525 -222
- package/src/session/blob-store.ts +89 -0
- package/src/session/internal/managed-session-scope.ts +36 -26
- package/src/session/internal/managed-session-storage.ts +303 -32
- package/src/session/internal/session-memory-sidecar.ts +2160 -0
- package/src/session/session-manager.ts +7027 -394
- package/src/session/session-storage.ts +1134 -11
- package/src/task/executor.ts +17 -4
- package/src/tools/bash.ts +109 -7
- package/src/tools/find.ts +55 -26
- package/src/tools/todo-contract.ts +40 -8
- package/src/tui/output-block.ts +11 -1
|
@@ -0,0 +1,2160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standalone sidecar primitives for bounded-memory cold-session offloading.
|
|
3
|
+
*
|
|
4
|
+
* This module is intentionally self-contained: it imports only Node built-ins and
|
|
5
|
+
* defines the reusable P2/P3/P4 primitives agreed in the approved memory spec —
|
|
6
|
+
* the retained-byte accountant, reducer disk-ref demotion guardrails, the
|
|
7
|
+
* `isDerivedSessionMemoryFile` lifecycle predicate, the disposable sidecar v2
|
|
8
|
+
* index schemas, the unsorted append-only bounded dictionary builder with
|
|
9
|
+
* duplicate-ID sidecar-ineligibility detection, fixed-size cache accounting,
|
|
10
|
+
* the anchored base digest + rolling tail checksum chain with commit validation
|
|
11
|
+
* and the five-class reopen classifier, the `latestModelChange` / TTSR
|
|
12
|
+
* latest-wins reducer semantics, and bounded parent→children / labels/pins
|
|
13
|
+
* descriptors.
|
|
14
|
+
*
|
|
15
|
+
* Transcript v5 remains authoritative and every sidecar index is disposable /
|
|
16
|
+
* rebuildable. No builder here requires the whole transcript resident in
|
|
17
|
+
* memory; all builders consume records one at a time and enforce their peak
|
|
18
|
+
* budgets as they go.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
|
|
23
|
+
// ============================================================================
|
|
24
|
+
// Budgets and fixed-size bounds
|
|
25
|
+
// ============================================================================
|
|
26
|
+
|
|
27
|
+
/** Provider-context materialization acceptance peak (AC1). */
|
|
28
|
+
export const SESSION_MEMORY_ACCEPTANCE_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
29
|
+
/** Steady-state retained-byte budget: 61.0625 MiB leaves the 64 MiB margin. */
|
|
30
|
+
export const SESSION_MEMORY_STEADY_STATE_BUDGET_BYTES = 61 * 1024 * 1024 + 64 * 1024;
|
|
31
|
+
|
|
32
|
+
/** Fixed 24 B resident disk-ref descriptor (ADR / A.1). */
|
|
33
|
+
export const DESCRIPTOR_BYTES = 24;
|
|
34
|
+
/** Conservative per-record object overhead (A.1). */
|
|
35
|
+
export const RECORD_OBJECT_OVERHEAD_BYTES = 48;
|
|
36
|
+
/** labels+pins steady-state budget (0.0625 MiB). */
|
|
37
|
+
export const LABELS_PINS_BUDGET_BYTES = 64 * 1024;
|
|
38
|
+
|
|
39
|
+
/** Reducer resident budget (4 MiB) — enforced by disk-ref demotion. */
|
|
40
|
+
export const REDUCER_BUDGET_BYTES = 4 * 1024 * 1024;
|
|
41
|
+
/** Guardrail: values over this are demoted to disk-ref descriptors. */
|
|
42
|
+
export const MAX_REDUCER_INLINE_BYTES = 4 * 1024;
|
|
43
|
+
/** Guardrail: max resident string length. */
|
|
44
|
+
export const MAX_REDUCER_STRING_BYTES = 4 * 1024;
|
|
45
|
+
/** Guardrail: max distinct roles. */
|
|
46
|
+
export const MAX_ROLES = 64;
|
|
47
|
+
/** Guardrail: max chain entries per role. */
|
|
48
|
+
export const MAX_CHAIN_ENTRIES = 32;
|
|
49
|
+
/** Guardrail: max TTSR rules. */
|
|
50
|
+
export const MAX_TTSR_RULES = 256;
|
|
51
|
+
/** Guardrail: max MCP names. */
|
|
52
|
+
export const MAX_MCP_NAMES = 4096;
|
|
53
|
+
/** Guardrail: max chars per MCP name. */
|
|
54
|
+
export const MAX_MCP_NAME_CHARS = 256;
|
|
55
|
+
/** Guardrail: modeData ≤ 64 KiB. */
|
|
56
|
+
export const MODE_DATA_MAX_BYTES = 64 * 1024;
|
|
57
|
+
/** Guardrail: compaction summary ≤ 64 KiB. */
|
|
58
|
+
export const COMPACTION_SUMMARY_MAX_BYTES = 64 * 1024;
|
|
59
|
+
/** preserveData.openaiRemoteCompaction ≤ 8 MiB (always a disk ref). */
|
|
60
|
+
export const OPENAI_REMOTE_COMPACTION_MAX_BYTES = 8 * 1024 * 1024;
|
|
61
|
+
|
|
62
|
+
/** Fixed-size index block cache (steady-state). */
|
|
63
|
+
export const BLOCK_CACHE_BUDGET_BYTES = 8 * 1024 * 1024;
|
|
64
|
+
/** Fixed-size resolved-entry cache (steady-state). */
|
|
65
|
+
export const ENTRY_CACHE_BUDGET_BYTES = 28 * 1024 * 1024;
|
|
66
|
+
/** Tail journal buffer/index (steady-state). */
|
|
67
|
+
export const TAIL_BUFFER_BUDGET_BYTES = 4 * 1024 * 1024;
|
|
68
|
+
/** Bounded forward-scan window for transcript-ahead recovery (R2.4). */
|
|
69
|
+
export const INDEX_TAIL_SCAN_MAX_BYTES = 4 * 1024 * 1024;
|
|
70
|
+
/** Maximum ordinal-index records retained transiently for one cold branch activation. */
|
|
71
|
+
export const COLD_BRANCH_PREFETCH_MAX_INDEX_ENTRIES = 16 * 1024;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Decide the cold-branch ordinal-run fast path before allocating its Map or
|
|
75
|
+
* transcript window. Large sparse/interleaved intervals fail closed to bounded
|
|
76
|
+
* dictionary/index fallback; the transcript remains authoritative.
|
|
77
|
+
*/
|
|
78
|
+
export function coldBranchOrdinalRunWithinPrefetchBounds(input: {
|
|
79
|
+
boundaryOrdinal: number;
|
|
80
|
+
leafOrdinal: number;
|
|
81
|
+
transcriptStart: number;
|
|
82
|
+
transcriptEnd: number;
|
|
83
|
+
maxTranscriptBytes: number;
|
|
84
|
+
}): boolean {
|
|
85
|
+
const values = [
|
|
86
|
+
input.boundaryOrdinal,
|
|
87
|
+
input.leafOrdinal,
|
|
88
|
+
input.transcriptStart,
|
|
89
|
+
input.transcriptEnd,
|
|
90
|
+
input.maxTranscriptBytes,
|
|
91
|
+
];
|
|
92
|
+
if (!values.every(value => Number.isSafeInteger(value))) return false;
|
|
93
|
+
const intervalEntries = input.leafOrdinal - input.boundaryOrdinal + 1;
|
|
94
|
+
const transcriptLength = input.transcriptEnd - input.transcriptStart;
|
|
95
|
+
return (
|
|
96
|
+
input.boundaryOrdinal >= 0 &&
|
|
97
|
+
intervalEntries > 0 &&
|
|
98
|
+
intervalEntries <= COLD_BRANCH_PREFETCH_MAX_INDEX_ENTRIES &&
|
|
99
|
+
input.transcriptStart >= 0 &&
|
|
100
|
+
transcriptLength > 0 &&
|
|
101
|
+
transcriptLength <= input.maxTranscriptBytes
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Dictionary build acceptance cap; default buffers stay well below this 20 MiB maximum. */
|
|
106
|
+
export const DICTIONARY_BUILD_PEAK_BYTES = 20 * 1024 * 1024;
|
|
107
|
+
/** One dictionary partition buffer. */
|
|
108
|
+
export const DICTIONARY_PARTITION_BUFFER_BYTES = 128 * 1024;
|
|
109
|
+
/** Dictionary bucket journal flush buffer. */
|
|
110
|
+
export const DICTIONARY_BUCKET_JOURNAL_BYTES = 128 * 1024;
|
|
111
|
+
|
|
112
|
+
/** Bounded parent→children index bounds. */
|
|
113
|
+
export const PARENT_CHILDREN_MAX_PARENTS = 4096;
|
|
114
|
+
export const PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT = 256;
|
|
115
|
+
export const PARENT_CHILDREN_BUDGET_BYTES = 4 * 1024 * 1024;
|
|
116
|
+
/** Deterministic hash-bucket count for the disposable on-disk parent artifact. */
|
|
117
|
+
export const PARENT_CHILDREN_BUCKET_COUNT = 64;
|
|
118
|
+
|
|
119
|
+
// ============================================================================
|
|
120
|
+
// Retained-byte formulas (A.1)
|
|
121
|
+
// ============================================================================
|
|
122
|
+
|
|
123
|
+
/** Resident string = `2 × charCount + 16 B`. */
|
|
124
|
+
export function residentStringBytes(value: string): number {
|
|
125
|
+
return 2 * value.length + 16;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Resident array = `8 B/slot + Σ elements`. */
|
|
129
|
+
export function residentArrayBytes(slotCount: number, elementBytes: number): number {
|
|
130
|
+
return 8 * slotCount + elementBytes;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Per-record object with `childBytes` of payload = 48 B overhead + payload. */
|
|
134
|
+
export function residentRecordBytes(childBytes: number): number {
|
|
135
|
+
return RECORD_OBJECT_OVERHEAD_BYTES + childBytes;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Authoritative measured retained-byte counter. Enforcement acts on the total,
|
|
140
|
+
* never on field counts alone. `tryCharge`/`wouldExceed` let callers enforce a
|
|
141
|
+
* budget without mutating state (used by the provider-context preflight which
|
|
142
|
+
* must never retain an over-budget graph).
|
|
143
|
+
*/
|
|
144
|
+
export class SessionMemoryAccountant {
|
|
145
|
+
private total = 0;
|
|
146
|
+
private readonly budget: number;
|
|
147
|
+
|
|
148
|
+
constructor(budgetBytes: number = SESSION_MEMORY_STEADY_STATE_BUDGET_BYTES) {
|
|
149
|
+
this.budget = budgetBytes;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
get totalBytes(): number {
|
|
153
|
+
return this.total;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
get budgetBytes(): number {
|
|
157
|
+
return this.budget;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
get remainingBytes(): number {
|
|
161
|
+
return Math.max(0, this.budget - this.total);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Adds `bytes` to the running total and returns the new total. */
|
|
165
|
+
charge(bytes: number): number {
|
|
166
|
+
this.assertBytes(bytes);
|
|
167
|
+
this.total += bytes;
|
|
168
|
+
return this.total;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Charges one resident string via the `2 × charCount + 16` formula. */
|
|
172
|
+
chargeString(value: string): number {
|
|
173
|
+
return this.charge(residentStringBytes(value));
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Charges one resident array via the `8 B/slot + Σ elements` formula. */
|
|
177
|
+
chargeArray(slotCount: number, elementBytes: number): number {
|
|
178
|
+
return this.charge(residentArrayBytes(slotCount, elementBytes));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Charges one fixed 24 B disk-ref descriptor. */
|
|
182
|
+
chargeDescriptor(): number {
|
|
183
|
+
return this.charge(DESCRIPTOR_BYTES);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Charges one per-record object (48 B overhead + payload). */
|
|
187
|
+
chargeRecord(childBytes: number): number {
|
|
188
|
+
return this.charge(residentRecordBytes(childBytes));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Adds `bytes` only if the running total stays within the budget.
|
|
193
|
+
* Returns `false` (and does not mutate) when the addition would exceed it.
|
|
194
|
+
*/
|
|
195
|
+
tryCharge(bytes: number): boolean {
|
|
196
|
+
this.assertBytes(bytes);
|
|
197
|
+
if (this.total + bytes > this.budget) return false;
|
|
198
|
+
this.total += bytes;
|
|
199
|
+
return true;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Returns `true` when charging `bytes` would exceed the budget. */
|
|
203
|
+
wouldExceed(bytes: number): boolean {
|
|
204
|
+
this.assertBytes(bytes);
|
|
205
|
+
return this.total + bytes > this.budget;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
isWithinBudget(): boolean {
|
|
209
|
+
return this.total <= this.budget;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
release(bytes: number): void {
|
|
213
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_release_bytes");
|
|
214
|
+
if (bytes > this.total) throw new RangeError("release_exceeds_charged");
|
|
215
|
+
this.total -= bytes;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
snapshot(): { totalBytes: number; budgetBytes: number; remainingBytes: number } {
|
|
219
|
+
return { totalBytes: this.total, budgetBytes: this.budget, remainingBytes: this.remainingBytes };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private assertBytes(bytes: number): void {
|
|
223
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_charge_bytes");
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ============================================================================
|
|
228
|
+
// Reducer disk-ref demotion guardrails
|
|
229
|
+
// ============================================================================
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* A 24 B resident descriptor that points at a value stored in the sidecar
|
|
233
|
+
* metadata-delta section. Demoted values are never resident as full strings.
|
|
234
|
+
*/
|
|
235
|
+
export interface ReducerDemotionDescriptor {
|
|
236
|
+
section: "metadata-delta";
|
|
237
|
+
key: string;
|
|
238
|
+
offset: number;
|
|
239
|
+
length: number;
|
|
240
|
+
sha256: string;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export interface ReducerValueSnapshot {
|
|
244
|
+
key: string;
|
|
245
|
+
kind: "inline" | "disk_ref";
|
|
246
|
+
residentBytes: number;
|
|
247
|
+
storedBytes: number;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export type ReducerBudgetResult =
|
|
251
|
+
| { kind: "ok"; totalBytes: number }
|
|
252
|
+
| { kind: "over_budget_irreducible"; totalBytes: number };
|
|
253
|
+
|
|
254
|
+
interface TrackedReducerValue {
|
|
255
|
+
key: string;
|
|
256
|
+
kind: "inline" | "disk_ref";
|
|
257
|
+
residentBytes: number;
|
|
258
|
+
storedBytes: number;
|
|
259
|
+
sha256: string;
|
|
260
|
+
metadataBytes: number;
|
|
261
|
+
offset: number;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Tracks the reducer's resident bytes against the 4 MiB budget. Values over
|
|
266
|
+
* `MAX_REDUCER_INLINE_BYTES` are immediately stored as 24 B disk-ref
|
|
267
|
+
* descriptors. When the total would exceed the budget, the largest resident
|
|
268
|
+
* inline value is demoted, repeating until the total fits or only
|
|
269
|
+
* descriptors/scalars remain (impossible by arithmetic — checked anyway).
|
|
270
|
+
*/
|
|
271
|
+
export class ReducerBudget {
|
|
272
|
+
private readonly values = new Map<string, TrackedReducerValue>();
|
|
273
|
+
private total = 0;
|
|
274
|
+
private nextOffset = 0;
|
|
275
|
+
private readonly budgetBytesValue: number;
|
|
276
|
+
private readonly maxInlineBytes: number;
|
|
277
|
+
|
|
278
|
+
constructor(budgetBytes: number = REDUCER_BUDGET_BYTES, maxInlineBytes: number = MAX_REDUCER_INLINE_BYTES) {
|
|
279
|
+
this.budgetBytesValue = budgetBytes;
|
|
280
|
+
this.maxInlineBytes = maxInlineBytes;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
get totalBytes(): number {
|
|
284
|
+
return this.total;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
get budgetBytes(): number {
|
|
288
|
+
return this.budgetBytesValue;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
get(key: string): ReducerValueSnapshot | undefined {
|
|
292
|
+
const value = this.values.get(key);
|
|
293
|
+
if (value === undefined) return undefined;
|
|
294
|
+
return { key: value.key, kind: value.kind, residentBytes: value.residentBytes, storedBytes: value.storedBytes };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Returns the resident 24 B descriptor for a demoted key, if any. */
|
|
298
|
+
getDescriptor(key: string): ReducerDemotionDescriptor | undefined {
|
|
299
|
+
const value = this.values.get(key);
|
|
300
|
+
if (value === undefined || value.kind !== "disk_ref") return undefined;
|
|
301
|
+
return {
|
|
302
|
+
section: "metadata-delta",
|
|
303
|
+
key: value.key,
|
|
304
|
+
offset: value.offset,
|
|
305
|
+
length: value.storedBytes,
|
|
306
|
+
sha256: value.sha256,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Records a reducer value of `storedBytes` bytes (optionally with its content
|
|
312
|
+
* digest) and enforces the budget by demoting the largest resident inline
|
|
313
|
+
* value until the total fits.
|
|
314
|
+
*/
|
|
315
|
+
setInline(key: string, storedBytes: number, sha256 = ""): ReducerBudgetResult {
|
|
316
|
+
if (!Number.isSafeInteger(storedBytes) || storedBytes < 0) throw new RangeError("invalid_reducer_value_bytes");
|
|
317
|
+
const previous = this.values.get(key);
|
|
318
|
+
if (previous !== undefined) {
|
|
319
|
+
this.total -= previous.residentBytes + previous.metadataBytes;
|
|
320
|
+
this.values.delete(key);
|
|
321
|
+
}
|
|
322
|
+
const metadataBytes = residentStringBytes(key) + residentStringBytes(sha256) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
323
|
+
if (storedBytes > this.maxInlineBytes) {
|
|
324
|
+
this.values.set(key, {
|
|
325
|
+
key,
|
|
326
|
+
kind: "disk_ref",
|
|
327
|
+
residentBytes: DESCRIPTOR_BYTES,
|
|
328
|
+
metadataBytes,
|
|
329
|
+
storedBytes,
|
|
330
|
+
sha256,
|
|
331
|
+
offset: this.nextOffset++,
|
|
332
|
+
});
|
|
333
|
+
this.total += DESCRIPTOR_BYTES + metadataBytes;
|
|
334
|
+
} else {
|
|
335
|
+
this.values.set(key, {
|
|
336
|
+
key,
|
|
337
|
+
kind: "inline",
|
|
338
|
+
residentBytes: storedBytes,
|
|
339
|
+
metadataBytes,
|
|
340
|
+
storedBytes,
|
|
341
|
+
sha256,
|
|
342
|
+
offset: this.nextOffset++,
|
|
343
|
+
});
|
|
344
|
+
this.total += storedBytes + metadataBytes;
|
|
345
|
+
}
|
|
346
|
+
return this.enforceBudget();
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Demotes every currently resident inline value to a disk-ref descriptor. */
|
|
350
|
+
demoteAll(): void {
|
|
351
|
+
for (const value of this.values.values()) {
|
|
352
|
+
if (value.kind === "inline" && value.residentBytes > DESCRIPTOR_BYTES) this.demote(value);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
private enforceBudget(): ReducerBudgetResult {
|
|
357
|
+
while (this.total > this.budgetBytesValue) {
|
|
358
|
+
const largest = this.largestDemotableInline();
|
|
359
|
+
if (largest === undefined) {
|
|
360
|
+
// Only descriptors/scalars remain but still over budget — impossible
|
|
361
|
+
// by arithmetic; the caller keeps the record hot and raises the
|
|
362
|
+
// token-compaction trigger. Never unbounded resident state.
|
|
363
|
+
return { kind: "over_budget_irreducible", totalBytes: this.total };
|
|
364
|
+
}
|
|
365
|
+
this.demote(largest);
|
|
366
|
+
}
|
|
367
|
+
return { kind: "ok", totalBytes: this.total };
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private largestDemotableInline(): TrackedReducerValue | undefined {
|
|
371
|
+
let best: TrackedReducerValue | undefined;
|
|
372
|
+
for (const value of this.values.values()) {
|
|
373
|
+
if (
|
|
374
|
+
value.kind === "inline" &&
|
|
375
|
+
value.residentBytes > DESCRIPTOR_BYTES &&
|
|
376
|
+
(best === undefined || value.residentBytes > best.residentBytes)
|
|
377
|
+
) {
|
|
378
|
+
best = value;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
return best;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
private demote(value: TrackedReducerValue): void {
|
|
385
|
+
const delta = value.residentBytes - DESCRIPTOR_BYTES;
|
|
386
|
+
value.residentBytes = DESCRIPTOR_BYTES;
|
|
387
|
+
value.kind = "disk_ref";
|
|
388
|
+
this.total -= delta;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// ============================================================================
|
|
393
|
+
// Derived-file lifecycle predicate (MEM-C-014/024)
|
|
394
|
+
// ============================================================================
|
|
395
|
+
|
|
396
|
+
const SPILL_SEGMENT = ".spill.";
|
|
397
|
+
const DERIVED_SUFFIXES = new Set(["idx", "tail", "commit", "buckets"]);
|
|
398
|
+
const DERIVED_PREFIXES = ["dict-", "capture-", "fork-", "overlay-", "parent-", "metadata-"];
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Single source of truth for every sidecar artifact and temp: `*.spill.idx`,
|
|
402
|
+
* `*.spill.tail`, `*.spill.commit`, `*.spill.buckets`, `*.spill.dict-*`,
|
|
403
|
+
* `*.spill.capture-*`, `*.spill.fork-*`, `*.spill.overlay-*`,
|
|
404
|
+
* `*.spill.parent-*` (persistent parent→children buckets),
|
|
405
|
+
* `*.spill.metadata-*` (persistent metadata-delta section), and any
|
|
406
|
+
* `*.spill.*.tmp`. Used at every fork/delete/sweep/Memory-parity callsite so
|
|
407
|
+
* ad-hoc filters cannot drift.
|
|
408
|
+
*/
|
|
409
|
+
export function isDerivedSessionMemoryFile(pathname: string): boolean {
|
|
410
|
+
const base = pathname.slice(Math.max(pathname.lastIndexOf("/"), pathname.lastIndexOf("\\")) + 1);
|
|
411
|
+
const idx = base.indexOf(SPILL_SEGMENT);
|
|
412
|
+
if (idx === -1) return false;
|
|
413
|
+
const rest = base.slice(idx + SPILL_SEGMENT.length);
|
|
414
|
+
if (rest.length === 0) return false;
|
|
415
|
+
if (DERIVED_SUFFIXES.has(rest)) return true;
|
|
416
|
+
for (const prefix of DERIVED_PREFIXES) {
|
|
417
|
+
if (rest.startsWith(prefix) && rest.length > prefix.length) return true;
|
|
418
|
+
}
|
|
419
|
+
if (rest.endsWith(".tmp") && rest.length > ".tmp".length) return true;
|
|
420
|
+
return false;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// ============================================================================
|
|
424
|
+
// Sidecar v2 disposable index schemas
|
|
425
|
+
// ============================================================================
|
|
426
|
+
|
|
427
|
+
/** Common header for every sidecar v2 index file. */
|
|
428
|
+
export interface SidecarIndexHeaderV2 {
|
|
429
|
+
version: 2;
|
|
430
|
+
sessionId: string;
|
|
431
|
+
/** Set when duplicate record IDs were detected; session is eager-ineligible. */
|
|
432
|
+
sidecarIneligible: boolean;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** Unsorted append-only variable-width dictionary (term → id). */
|
|
436
|
+
export interface SidecarDictionaryV2 {
|
|
437
|
+
header: SidecarIndexHeaderV2;
|
|
438
|
+
/** Insertion-ordered unique terms (append-only; deterministic). */
|
|
439
|
+
terms: readonly string[];
|
|
440
|
+
/** term → dictionary id, assigned by first-insertion order. */
|
|
441
|
+
idByTerm: ReadonlyMap<string, number>;
|
|
442
|
+
/** Combined serialized term bytes. */
|
|
443
|
+
totalBytes: number;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Rolling tail checksum index (anchored base + terminal chain). */
|
|
447
|
+
export interface SidecarTailIndexV2 {
|
|
448
|
+
header: SidecarIndexHeaderV2;
|
|
449
|
+
base: BaseAnchor;
|
|
450
|
+
records: readonly TailRecord[];
|
|
451
|
+
terminalChecksum: string;
|
|
452
|
+
terminalSeq: number;
|
|
453
|
+
transcriptSize: number;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** Metadata-delta section: demoted reducer values + deltas. */
|
|
457
|
+
export interface SidecarMetadataIndexV2 {
|
|
458
|
+
header: SidecarIndexHeaderV2;
|
|
459
|
+
/** key → { offset, length, sha256 } for demoted values. */
|
|
460
|
+
entries: Record<string, { offset: number; length: number; sha256: string }>;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Bounded parent→children index. */
|
|
464
|
+
export interface SidecarParentIndexV2 {
|
|
465
|
+
header: SidecarIndexHeaderV2;
|
|
466
|
+
entries: readonly ParentChildrenIndexEntry[];
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** Bounded labels/pins descriptor. */
|
|
470
|
+
export interface SidecarLabelsPinsIndexV2 {
|
|
471
|
+
header: SidecarIndexHeaderV2;
|
|
472
|
+
labels: ReadonlyMap<string, string>;
|
|
473
|
+
pins: ReadonlyMap<string, string>;
|
|
474
|
+
totalBytes: number;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ============================================================================
|
|
478
|
+
// Bounded dictionary builder (20 MiB peak, duplicate-ID detection)
|
|
479
|
+
// ============================================================================
|
|
480
|
+
|
|
481
|
+
export interface DictionaryRecordInput {
|
|
482
|
+
ordinal: number;
|
|
483
|
+
id: string;
|
|
484
|
+
bytes: Uint8Array;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
export interface DictionaryBuildOptions {
|
|
488
|
+
peakBudgetBytes?: number;
|
|
489
|
+
partitionBufferBytes?: number;
|
|
490
|
+
bucketJournalBytes?: number;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
export type DictionaryAddResult = { kind: "ok" } | { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number };
|
|
494
|
+
|
|
495
|
+
export type DictionaryBuildResult =
|
|
496
|
+
| { kind: "ok"; dictionary: SidecarDictionaryV2; stats: DictionaryBuildStats }
|
|
497
|
+
| { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number };
|
|
498
|
+
|
|
499
|
+
export interface DictionaryBuildStats {
|
|
500
|
+
totalRecords: number;
|
|
501
|
+
uniqueTerms: number;
|
|
502
|
+
totalBytes: number;
|
|
503
|
+
peakBytes: number;
|
|
504
|
+
duplicateIds: readonly string[];
|
|
505
|
+
sidecarIneligible: boolean;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
const utf8Decoder = new TextDecoder("utf-8");
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* Unsorted append-only variable-width dictionary builder. Consumes records one
|
|
512
|
+
* at a time (no whole transcript resident) and tracks the 20 MiB build peak as
|
|
513
|
+
* `Σ term bytes + 4 × partition buffer + bucket journal`. Duplicate record IDs
|
|
514
|
+
* mark the session sidecar-ineligible (eager fallback) while the build still
|
|
515
|
+
* completes deterministically.
|
|
516
|
+
*/
|
|
517
|
+
export class BoundedDictionaryBuilder {
|
|
518
|
+
private readonly terms: string[] = [];
|
|
519
|
+
private readonly idByTerm = new Map<string, number>();
|
|
520
|
+
private readonly seenRecordIds = new Map<string, number>();
|
|
521
|
+
private readonly duplicateIds: string[] = [];
|
|
522
|
+
private totalTermBytes = 0;
|
|
523
|
+
private recordIdBytes = 0;
|
|
524
|
+
private duplicateIdBytes = 0;
|
|
525
|
+
private peakBytes = 0;
|
|
526
|
+
private sidecarIneligible = false;
|
|
527
|
+
private recordCount = 0;
|
|
528
|
+
private finished = false;
|
|
529
|
+
private readonly peakBudget: number;
|
|
530
|
+
private readonly partitionBase: number;
|
|
531
|
+
|
|
532
|
+
constructor(options: DictionaryBuildOptions = {}) {
|
|
533
|
+
this.peakBudget = options.peakBudgetBytes ?? DICTIONARY_BUILD_PEAK_BYTES;
|
|
534
|
+
const partition = options.partitionBufferBytes ?? DICTIONARY_PARTITION_BUFFER_BYTES;
|
|
535
|
+
const journal = options.bucketJournalBytes ?? DICTIONARY_BUCKET_JOURNAL_BYTES;
|
|
536
|
+
this.partitionBase = 4 * partition + journal;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
add(record: DictionaryRecordInput): DictionaryAddResult {
|
|
540
|
+
if (this.finished) throw new Error("dictionary_builder_finished");
|
|
541
|
+
this.recordCount += 1;
|
|
542
|
+
const priorOrdinal = this.seenRecordIds.get(record.id);
|
|
543
|
+
if (priorOrdinal !== undefined) {
|
|
544
|
+
this.sidecarIneligible = true;
|
|
545
|
+
if (!this.duplicateIds.includes(record.id)) {
|
|
546
|
+
this.duplicateIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
547
|
+
this.duplicateIds.push(record.id);
|
|
548
|
+
}
|
|
549
|
+
} else {
|
|
550
|
+
this.recordIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
551
|
+
this.seenRecordIds.set(record.id, record.ordinal);
|
|
552
|
+
}
|
|
553
|
+
this.updatePeak();
|
|
554
|
+
if (this.peakBytes > this.peakBudget) {
|
|
555
|
+
return { kind: "budget_exceeded", peakBytes: this.peakBytes, budgetBytes: this.peakBudget };
|
|
556
|
+
}
|
|
557
|
+
const term = utf8Decoder.decode(record.bytes);
|
|
558
|
+
if (!this.idByTerm.has(term)) {
|
|
559
|
+
this.idByTerm.set(term, this.idByTerm.size);
|
|
560
|
+
this.terms.push(term);
|
|
561
|
+
this.totalTermBytes += residentStringBytes(term) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
562
|
+
this.updatePeak();
|
|
563
|
+
}
|
|
564
|
+
if (this.peakBytes > this.peakBudget) {
|
|
565
|
+
return { kind: "budget_exceeded", peakBytes: this.peakBytes, budgetBytes: this.peakBudget };
|
|
566
|
+
}
|
|
567
|
+
return { kind: "ok" };
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
finish(): DictionaryBuildResult {
|
|
571
|
+
if (this.finished) throw new Error("dictionary_builder_already_finished");
|
|
572
|
+
this.finished = true;
|
|
573
|
+
return {
|
|
574
|
+
kind: "ok",
|
|
575
|
+
dictionary: {
|
|
576
|
+
header: { version: 2, sessionId: "", sidecarIneligible: this.sidecarIneligible },
|
|
577
|
+
terms: this.terms,
|
|
578
|
+
idByTerm: this.idByTerm,
|
|
579
|
+
totalBytes: this.totalTermBytes,
|
|
580
|
+
},
|
|
581
|
+
stats: {
|
|
582
|
+
totalRecords: this.recordCount,
|
|
583
|
+
uniqueTerms: this.terms.length,
|
|
584
|
+
totalBytes: this.totalTermBytes,
|
|
585
|
+
peakBytes: this.peakBytes,
|
|
586
|
+
duplicateIds: this.duplicateIds,
|
|
587
|
+
sidecarIneligible: this.sidecarIneligible,
|
|
588
|
+
},
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
private updatePeak(): void {
|
|
593
|
+
const current = Math.max(this.partitionBase, this.totalTermBytes + this.recordIdBytes + this.duplicateIdBytes);
|
|
594
|
+
if (current > this.peakBytes) this.peakBytes = current;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// ============================================================================
|
|
599
|
+
// Fixed-size cache accounting (block / entry / tail)
|
|
600
|
+
// ============================================================================
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Tracks a fixed-size cache budget (8 MiB block, 28 MiB entry, 4 MiB tail).
|
|
604
|
+
* `tryAllocate` rejects allocations that would exceed the budget without
|
|
605
|
+
* mutating state.
|
|
606
|
+
*/
|
|
607
|
+
export class FixedCacheAccount {
|
|
608
|
+
private allocated = 0;
|
|
609
|
+
private readonly budget: number;
|
|
610
|
+
|
|
611
|
+
constructor(budgetBytes: number) {
|
|
612
|
+
this.budget = budgetBytes;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
get allocatedBytes(): number {
|
|
616
|
+
return this.allocated;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
get budgetBytes(): number {
|
|
620
|
+
return this.budget;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
get remainingBytes(): number {
|
|
624
|
+
return Math.max(0, this.budget - this.allocated);
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
tryAllocate(bytes: number): boolean {
|
|
628
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_cache_allocation");
|
|
629
|
+
if (this.allocated + bytes > this.budget) return false;
|
|
630
|
+
this.allocated += bytes;
|
|
631
|
+
return true;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
release(bytes: number): void {
|
|
635
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError("invalid_cache_release");
|
|
636
|
+
if (bytes > this.allocated) throw new RangeError("cache_release_exceeds_allocated");
|
|
637
|
+
this.allocated -= bytes;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// ============================================================================
|
|
642
|
+
// Anchored base digest + rolling tail checksum chain (C.1)
|
|
643
|
+
// ============================================================================
|
|
644
|
+
|
|
645
|
+
/** Anchors the base range `[0, baseEndOffset)` with a content digest. */
|
|
646
|
+
export interface BaseAnchor {
|
|
647
|
+
baseDigest: string;
|
|
648
|
+
baseEndOffset: number;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
export type TailRecordKind = "user" | "assistant" | "tool" | "model_change" | "ttsr_injection" | "internal" | "other";
|
|
652
|
+
|
|
653
|
+
/** One committed rolling tail record. */
|
|
654
|
+
export interface TailRecord {
|
|
655
|
+
gen: number;
|
|
656
|
+
seq: number;
|
|
657
|
+
kind: TailRecordKind;
|
|
658
|
+
ordinal: number;
|
|
659
|
+
id: string;
|
|
660
|
+
parentId: string | null;
|
|
661
|
+
type: string;
|
|
662
|
+
byteOffset: number;
|
|
663
|
+
byteLength: number;
|
|
664
|
+
recordDigest: string;
|
|
665
|
+
checksum: string;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** Input for appending to a rolling tail chain (checksum is computed). */
|
|
669
|
+
export interface TailRecordInput {
|
|
670
|
+
gen?: number;
|
|
671
|
+
seq: number;
|
|
672
|
+
kind: TailRecordKind;
|
|
673
|
+
ordinal: number;
|
|
674
|
+
id: string;
|
|
675
|
+
parentId: string | null;
|
|
676
|
+
type: string;
|
|
677
|
+
byteOffset: number;
|
|
678
|
+
byteLength: number;
|
|
679
|
+
recordDigest: string;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/** Effective base+tail identity (no per-append whole-file hash). */
|
|
683
|
+
export interface CommittedTail {
|
|
684
|
+
base: BaseAnchor;
|
|
685
|
+
records: readonly TailRecord[];
|
|
686
|
+
terminalChecksum: string;
|
|
687
|
+
terminalSeq: number;
|
|
688
|
+
transcriptSize: number;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/** SHA-256 of one record line's bytes. */
|
|
692
|
+
export function computeLineDigest(bytes: Uint8Array): string {
|
|
693
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Canonical length-prefixed chain hash. Each part is hashed as its UTF-8 /
|
|
698
|
+
* raw bytes prefixed by a 4-byte big-endian length, so concatenations are
|
|
699
|
+
* unambiguous.
|
|
700
|
+
*/
|
|
701
|
+
export function hashChain(parts: ReadonlyArray<bigint | number | string | Uint8Array>): string {
|
|
702
|
+
const hash = createHash("sha256");
|
|
703
|
+
for (const part of parts) {
|
|
704
|
+
const bytes =
|
|
705
|
+
typeof part === "string"
|
|
706
|
+
? Buffer.from(part, "utf8")
|
|
707
|
+
: typeof part === "number"
|
|
708
|
+
? Buffer.from(String(part), "utf8")
|
|
709
|
+
: typeof part === "bigint"
|
|
710
|
+
? Buffer.from(part.toString(), "utf8")
|
|
711
|
+
: Buffer.from(part);
|
|
712
|
+
const length = Buffer.allocUnsafe(4);
|
|
713
|
+
length.writeUInt32BE(bytes.byteLength, 0);
|
|
714
|
+
hash.update(length).update(bytes);
|
|
715
|
+
}
|
|
716
|
+
return hash.digest("hex");
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** First tail checksum: `C0 = H(baseDigest ‖ baseEndOffset ‖ seq0 ‖ lineDigest0)`. */
|
|
720
|
+
export function computeC0(base: BaseAnchor, seq0: number, lineDigest0: string): string {
|
|
721
|
+
return hashChain([base.baseDigest, base.baseEndOffset, seq0, lineDigest0]);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** Checksum one complete tail record, including every field used for ordering or lookup. */
|
|
725
|
+
export function computeTailRecordChecksum(
|
|
726
|
+
base: BaseAnchor,
|
|
727
|
+
previousChecksum: string | undefined,
|
|
728
|
+
record: TailRecordInput & { gen?: number },
|
|
729
|
+
): string {
|
|
730
|
+
const fields: Array<string | number> = [
|
|
731
|
+
record.gen ?? 0,
|
|
732
|
+
record.seq,
|
|
733
|
+
record.kind,
|
|
734
|
+
record.ordinal,
|
|
735
|
+
record.id,
|
|
736
|
+
record.parentId === null ? "<null>" : record.parentId,
|
|
737
|
+
record.type,
|
|
738
|
+
record.byteOffset,
|
|
739
|
+
record.byteLength,
|
|
740
|
+
record.recordDigest,
|
|
741
|
+
];
|
|
742
|
+
return previousChecksum === undefined
|
|
743
|
+
? hashChain([base.baseDigest, base.baseEndOffset, ...fields])
|
|
744
|
+
: hashChain([previousChecksum, ...fields]);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/** Resident bytes for every retained tail string plus the record object. */
|
|
748
|
+
export function tailRecordResidentBytes(record: TailRecordInput, checksumChars = 64): number {
|
|
749
|
+
return residentRecordBytes(
|
|
750
|
+
residentStringBytes(record.id) +
|
|
751
|
+
(record.parentId === null ? 0 : residentStringBytes(record.parentId)) +
|
|
752
|
+
residentStringBytes(record.type) +
|
|
753
|
+
residentStringBytes(record.kind) +
|
|
754
|
+
residentStringBytes(record.recordDigest) +
|
|
755
|
+
(2 * checksumChars + 16),
|
|
756
|
+
);
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/** Terminal checksum for a committed tail (empty chain → base only). */
|
|
760
|
+
export function computeTerminalChecksum(base: BaseAnchor, records: readonly TailRecord[]): string {
|
|
761
|
+
if (records.length === 0) return hashChain([base.baseDigest, base.baseEndOffset]);
|
|
762
|
+
return records[records.length - 1].checksum;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
export interface TailValidationResult {
|
|
766
|
+
valid: boolean;
|
|
767
|
+
firstInvalidSeq?: number;
|
|
768
|
+
reason?: string;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* Validates a committed tail chain: contiguous seq from 0, contiguous absolute
|
|
773
|
+
* offsets from `baseEndOffset`, and checksum continuity
|
|
774
|
+
* `checksum_i = H(checksum_{i-1} ‖ seq_i ‖ offset_i ‖ length_i ‖ lineDigest_i)`.
|
|
775
|
+
*/
|
|
776
|
+
export function validateTailChain(base: BaseAnchor, records: readonly TailRecord[]): TailValidationResult {
|
|
777
|
+
let expectedSeq = 0;
|
|
778
|
+
let expectedOffset = base.baseEndOffset;
|
|
779
|
+
let previousChecksum: string | undefined;
|
|
780
|
+
for (const record of records) {
|
|
781
|
+
if (record.seq !== expectedSeq) {
|
|
782
|
+
return { valid: false, firstInvalidSeq: record.seq, reason: "seq_discontinuity" };
|
|
783
|
+
}
|
|
784
|
+
if (record.byteOffset !== expectedOffset) {
|
|
785
|
+
return { valid: false, firstInvalidSeq: record.seq, reason: "offset_discontinuity" };
|
|
786
|
+
}
|
|
787
|
+
const expectedChecksum = computeTailRecordChecksum(base, previousChecksum, record);
|
|
788
|
+
if (record.checksum !== expectedChecksum) {
|
|
789
|
+
return { valid: false, firstInvalidSeq: record.seq, reason: "checksum_mismatch" };
|
|
790
|
+
}
|
|
791
|
+
previousChecksum = expectedChecksum;
|
|
792
|
+
expectedSeq += 1;
|
|
793
|
+
expectedOffset = record.byteOffset + record.byteLength;
|
|
794
|
+
}
|
|
795
|
+
return { valid: true };
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Streams tail records, computing the rolling checksum chain incrementally and
|
|
800
|
+
* enforcing the 4 MiB tail buffer budget. Never buffers the whole transcript.
|
|
801
|
+
*/
|
|
802
|
+
export class RollingTailChainBuilder {
|
|
803
|
+
private readonly records: TailRecord[] = [];
|
|
804
|
+
private readonly base: BaseAnchor;
|
|
805
|
+
private lastChecksum: string | undefined;
|
|
806
|
+
private lastSeq = -1;
|
|
807
|
+
private lastEndOffset: number;
|
|
808
|
+
private remainingTailBytes: number;
|
|
809
|
+
|
|
810
|
+
constructor(base: BaseAnchor, options: { tailBufferBytes?: number } = {}) {
|
|
811
|
+
this.base = base;
|
|
812
|
+
this.lastEndOffset = base.baseEndOffset;
|
|
813
|
+
this.remainingTailBytes = options.tailBufferBytes ?? TAIL_BUFFER_BUDGET_BYTES;
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
get recordCount(): number {
|
|
817
|
+
return this.records.length;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/** Appends one tail record; returns `undefined` when the buffer budget is hit. */
|
|
821
|
+
append(input: TailRecordInput): TailRecord | undefined {
|
|
822
|
+
const expectedSeq = this.lastSeq + 1;
|
|
823
|
+
if (input.seq !== expectedSeq) throw new Error("tail_seq_discontinuity");
|
|
824
|
+
if (input.byteOffset !== this.lastEndOffset) throw new Error("tail_offset_discontinuity");
|
|
825
|
+
const checksum = computeTailRecordChecksum(this.base, this.lastChecksum, input);
|
|
826
|
+
const residentBytes = tailRecordResidentBytes(input, checksum.length);
|
|
827
|
+
if (residentBytes > this.remainingTailBytes) return undefined;
|
|
828
|
+
this.remainingTailBytes -= residentBytes;
|
|
829
|
+
const record: TailRecord = { ...input, gen: input.gen ?? 0, checksum };
|
|
830
|
+
this.records.push(record);
|
|
831
|
+
this.lastSeq = input.seq;
|
|
832
|
+
this.lastEndOffset = input.byteOffset + input.byteLength;
|
|
833
|
+
this.lastChecksum = checksum;
|
|
834
|
+
return record;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
build(): CommittedTail {
|
|
838
|
+
return {
|
|
839
|
+
base: this.base,
|
|
840
|
+
records: this.records,
|
|
841
|
+
terminalChecksum: computeTerminalChecksum(this.base, this.records),
|
|
842
|
+
terminalSeq: this.lastSeq,
|
|
843
|
+
transcriptSize: this.lastEndOffset,
|
|
844
|
+
};
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// ============================================================================
|
|
849
|
+
// Commit validation + five-class reopen classifier (R2.4)
|
|
850
|
+
// ============================================================================
|
|
851
|
+
|
|
852
|
+
/** Descriptor snapshot captured post-fsync from the authoritative open handle. */
|
|
853
|
+
export interface DescriptorSnapshot {
|
|
854
|
+
dev: bigint;
|
|
855
|
+
ino: bigint;
|
|
856
|
+
nlink?: bigint;
|
|
857
|
+
size: number;
|
|
858
|
+
mtimeNs: bigint;
|
|
859
|
+
ctimeNs: bigint;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/** Exact enum-order descriptor comparison (dev/ino/nlink/size/mtimeNs/ctimeNs). */
|
|
863
|
+
export function sameDescriptor(left: DescriptorSnapshot, right: DescriptorSnapshot): boolean {
|
|
864
|
+
return (
|
|
865
|
+
left.dev === right.dev &&
|
|
866
|
+
left.ino === right.ino &&
|
|
867
|
+
(left.nlink ?? 0n) === (right.nlink ?? 0n) &&
|
|
868
|
+
left.size === right.size &&
|
|
869
|
+
left.mtimeNs === right.mtimeNs &&
|
|
870
|
+
left.ctimeNs === right.ctimeNs
|
|
871
|
+
);
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/** Contents of the mutable `.spill.commit` marker (R2). */
|
|
875
|
+
export interface CommitMarkerContents {
|
|
876
|
+
gen: number;
|
|
877
|
+
descriptor: DescriptorSnapshot;
|
|
878
|
+
base: BaseAnchor;
|
|
879
|
+
terminalChecksum: string;
|
|
880
|
+
terminalSeq: number;
|
|
881
|
+
transcriptSize: number;
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
export type CommitInvalidReason =
|
|
885
|
+
| "missing_fields"
|
|
886
|
+
| "base_invalid"
|
|
887
|
+
| "chain_invalid"
|
|
888
|
+
| "terminal_checksum_mismatch"
|
|
889
|
+
| "terminal_seq_mismatch"
|
|
890
|
+
| "transcript_size_mismatch"
|
|
891
|
+
| "descriptor_mismatch"
|
|
892
|
+
| "terminal_marker_mismatch";
|
|
893
|
+
|
|
894
|
+
export type CommitValidationResult = { kind: "valid" } | { kind: "invalid"; reason: CommitInvalidReason };
|
|
895
|
+
|
|
896
|
+
/**
|
|
897
|
+
* Validates a commit marker's internal consistency plus its descriptor against
|
|
898
|
+
* the currently opened transcript. `baseValid`/`tailValid`/`terminalMarkerValid`
|
|
899
|
+
* are the externally-observed proof results.
|
|
900
|
+
*/
|
|
901
|
+
export function validateCommit(
|
|
902
|
+
commit: Partial<CommitMarkerContents> | CommitMarkerContents,
|
|
903
|
+
records: readonly TailRecord[],
|
|
904
|
+
observed: { descriptor: DescriptorSnapshot; baseValid: boolean; tailValid: boolean; terminalMarkerValid: boolean },
|
|
905
|
+
): CommitValidationResult {
|
|
906
|
+
if (commit === null || typeof commit !== "object") return { kind: "invalid", reason: "missing_fields" };
|
|
907
|
+
if (
|
|
908
|
+
typeof commit.gen !== "number" ||
|
|
909
|
+
commit.descriptor === undefined ||
|
|
910
|
+
commit.base === undefined ||
|
|
911
|
+
typeof commit.terminalChecksum !== "string" ||
|
|
912
|
+
typeof commit.terminalSeq !== "number" ||
|
|
913
|
+
typeof commit.transcriptSize !== "number"
|
|
914
|
+
) {
|
|
915
|
+
return { kind: "invalid", reason: "missing_fields" };
|
|
916
|
+
}
|
|
917
|
+
if (!observed.baseValid) return { kind: "invalid", reason: "base_invalid" };
|
|
918
|
+
const chain = validateTailChain(commit.base, records);
|
|
919
|
+
if (!chain.valid) return { kind: "invalid", reason: "chain_invalid" };
|
|
920
|
+
const computedTerminal = computeTerminalChecksum(commit.base, records);
|
|
921
|
+
if (commit.terminalChecksum !== computedTerminal) return { kind: "invalid", reason: "terminal_checksum_mismatch" };
|
|
922
|
+
const expectedTerminalSeq = records.length === 0 ? -1 : records[records.length - 1].seq;
|
|
923
|
+
if (commit.terminalSeq !== expectedTerminalSeq) return { kind: "invalid", reason: "terminal_seq_mismatch" };
|
|
924
|
+
const expectedSize =
|
|
925
|
+
records.length === 0
|
|
926
|
+
? commit.base.baseEndOffset
|
|
927
|
+
: records[records.length - 1].byteOffset + records[records.length - 1].byteLength;
|
|
928
|
+
if (commit.transcriptSize !== expectedSize) return { kind: "invalid", reason: "transcript_size_mismatch" };
|
|
929
|
+
if (!sameDescriptor(commit.descriptor, observed.descriptor))
|
|
930
|
+
return { kind: "invalid", reason: "descriptor_mismatch" };
|
|
931
|
+
if (!observed.terminalMarkerValid) return { kind: "invalid", reason: "terminal_marker_mismatch" };
|
|
932
|
+
return { kind: "valid" };
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
export type ReopenClassKind = "exact" | "transcript_ahead" | "tail_ahead" | "rebuild" | "stale_commit";
|
|
936
|
+
|
|
937
|
+
/** Physical evidence the reopen classifier consumes. */
|
|
938
|
+
export interface ReopenEvidence {
|
|
939
|
+
markerPresent: boolean;
|
|
940
|
+
descriptorExact: boolean;
|
|
941
|
+
sameObject: boolean;
|
|
942
|
+
sameSize: boolean;
|
|
943
|
+
sizeGrew: boolean;
|
|
944
|
+
sizeShrank: boolean;
|
|
945
|
+
withinScanWindow: boolean;
|
|
946
|
+
timesAdvanced: boolean;
|
|
947
|
+
timesChanged: boolean;
|
|
948
|
+
baseValid: boolean;
|
|
949
|
+
tailValid: boolean;
|
|
950
|
+
terminalMarkerValid: boolean;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
export interface ReopenClassification {
|
|
954
|
+
kind: ReopenClassKind;
|
|
955
|
+
reason: string;
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* Five-class reopen classification (R2.4). Deterministic over the supplied
|
|
960
|
+
* evidence; a stale commit is never accepted as current on any path.
|
|
961
|
+
*/
|
|
962
|
+
export function classifyReopen(evidence: ReopenEvidence): ReopenClassification {
|
|
963
|
+
if (!evidence.markerPresent) return { kind: "stale_commit", reason: "no_commit_marker" };
|
|
964
|
+
if (evidence.descriptorExact) {
|
|
965
|
+
if (evidence.baseValid && evidence.tailValid && evidence.terminalMarkerValid) {
|
|
966
|
+
return { kind: "exact", reason: "descriptor_and_proof_match" };
|
|
967
|
+
}
|
|
968
|
+
return { kind: "rebuild", reason: "descriptor_exact_proof_invalid" };
|
|
969
|
+
}
|
|
970
|
+
if (evidence.sameObject && evidence.sameSize && evidence.timesChanged) {
|
|
971
|
+
return { kind: "rebuild", reason: "same_size_mutation" };
|
|
972
|
+
}
|
|
973
|
+
if (evidence.sameObject && evidence.sameSize) {
|
|
974
|
+
if (evidence.baseValid && !evidence.tailValid)
|
|
975
|
+
return { kind: "tail_ahead", reason: "committed_tail_record_invalid" };
|
|
976
|
+
if (evidence.baseValid && evidence.tailValid && evidence.terminalMarkerValid) {
|
|
977
|
+
return { kind: "exact", reason: "descriptor_equivalent" };
|
|
978
|
+
}
|
|
979
|
+
return { kind: "rebuild", reason: "proof_invalid" };
|
|
980
|
+
}
|
|
981
|
+
if (evidence.sameObject && evidence.sizeGrew) {
|
|
982
|
+
if (evidence.withinScanWindow && evidence.timesAdvanced && evidence.baseValid && evidence.tailValid) {
|
|
983
|
+
return { kind: "transcript_ahead", reason: "object_grew_within_window" };
|
|
984
|
+
}
|
|
985
|
+
return { kind: "rebuild", reason: "over_window_or_invalid_base" };
|
|
986
|
+
}
|
|
987
|
+
if (evidence.sameObject && evidence.sizeShrank) {
|
|
988
|
+
return { kind: "rebuild", reason: "shrink_outside_tail_state" };
|
|
989
|
+
}
|
|
990
|
+
return { kind: "stale_commit", reason: "object_identity_mismatch" };
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
// ============================================================================
|
|
994
|
+
// Reducer deltas: nearest model-change role (R1) + TTSR latest-wins
|
|
995
|
+
// ============================================================================
|
|
996
|
+
|
|
997
|
+
export interface LatestModelChangeDelta {
|
|
998
|
+
kind: "latest_model_change";
|
|
999
|
+
ordinal: number;
|
|
1000
|
+
role?: string;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
export interface TtsrDelta {
|
|
1004
|
+
kind: "ttsr_injection";
|
|
1005
|
+
ordinal: number;
|
|
1006
|
+
rulesCount: number;
|
|
1007
|
+
recordsCount: number;
|
|
1008
|
+
count: number;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
export type ReducerDelta = LatestModelChangeDelta | TtsrDelta;
|
|
1012
|
+
|
|
1013
|
+
export interface ModelChangeReducerState {
|
|
1014
|
+
latest: { ordinal: number; role: string | undefined } | undefined;
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
export interface TtsrReducerState {
|
|
1018
|
+
count: number;
|
|
1019
|
+
rulesCount: number;
|
|
1020
|
+
recordsCount: number;
|
|
1021
|
+
largestOrdinal: number;
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
export interface ReducerState {
|
|
1025
|
+
modelChange: ModelChangeReducerState;
|
|
1026
|
+
ttsr: TtsrReducerState;
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
export function createReducerState(): ReducerState {
|
|
1030
|
+
return {
|
|
1031
|
+
modelChange: { latest: undefined },
|
|
1032
|
+
ttsr: { count: 0, rulesCount: 0, recordsCount: 0, largestOrdinal: -1 },
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
/**
|
|
1037
|
+
* Applies one reducer delta. `latest_model_change` overwrites the nearest
|
|
1038
|
+
* event's `{ordinal, role}`; `ttsr_injection.count` replaces the prior value
|
|
1039
|
+
* (authoritative `buildSessionContext`, latest-wins).
|
|
1040
|
+
*/
|
|
1041
|
+
export function applyReducerDelta(state: ReducerState, delta: ReducerDelta): ReducerState {
|
|
1042
|
+
switch (delta.kind) {
|
|
1043
|
+
case "latest_model_change":
|
|
1044
|
+
return { ...state, modelChange: { latest: { ordinal: delta.ordinal, role: delta.role } } };
|
|
1045
|
+
case "ttsr_injection":
|
|
1046
|
+
return {
|
|
1047
|
+
...state,
|
|
1048
|
+
ttsr: {
|
|
1049
|
+
count: delta.count,
|
|
1050
|
+
rulesCount: delta.rulesCount,
|
|
1051
|
+
recordsCount: delta.recordsCount,
|
|
1052
|
+
largestOrdinal: Math.max(state.ttsr.largestOrdinal, delta.ordinal),
|
|
1053
|
+
},
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* R1 consumer: returns `latestModelChange.role ?? "default"` when a
|
|
1060
|
+
* `model_change` exists on the path, else `undefined`. `hasExplicitDefaultModel`
|
|
1061
|
+
* only gates legacy assistant inference into `models.default`; it never feeds
|
|
1062
|
+
* this getter.
|
|
1063
|
+
*/
|
|
1064
|
+
export function getLastModelChangeRole(state: ReducerState): string | undefined {
|
|
1065
|
+
const latest = state.modelChange.latest;
|
|
1066
|
+
if (latest === undefined) return undefined;
|
|
1067
|
+
return latest.role ?? "default";
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/**
|
|
1071
|
+
* Compaction fold: keeps the max-ordinal `model_change` (nearest wins) and the
|
|
1072
|
+
* latest-ordinal TTSR state, matching the eager leaf→root first-hit semantics.
|
|
1073
|
+
*/
|
|
1074
|
+
export function foldReducerStates(left: ReducerState, right: ReducerState): ReducerState {
|
|
1075
|
+
const modelChange = foldModelChange(left.modelChange.latest, right.modelChange.latest);
|
|
1076
|
+
const ttsr = left.ttsr.largestOrdinal >= right.ttsr.largestOrdinal ? left.ttsr : right.ttsr;
|
|
1077
|
+
return { modelChange: { latest: modelChange }, ttsr };
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
function foldModelChange(
|
|
1081
|
+
left: ModelChangeReducerState["latest"],
|
|
1082
|
+
right: ModelChangeReducerState["latest"],
|
|
1083
|
+
): ModelChangeReducerState["latest"] {
|
|
1084
|
+
if (left === undefined) return right;
|
|
1085
|
+
if (right === undefined) return left;
|
|
1086
|
+
return left.ordinal >= right.ordinal ? left : right;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// ============================================================================
|
|
1090
|
+
// Bounded parent→children and labels/pins descriptors
|
|
1091
|
+
// ============================================================================
|
|
1092
|
+
|
|
1093
|
+
export interface ParentChildrenIndexEntry {
|
|
1094
|
+
parentId: string;
|
|
1095
|
+
children: readonly string[];
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/** Bounded parent→children index (rejects when either bound is exceeded). */
|
|
1099
|
+
export class BoundedParentChildrenIndex {
|
|
1100
|
+
private readonly childrenByParent = new Map<string, string[]>();
|
|
1101
|
+
private readonly maxParents: number;
|
|
1102
|
+
private readonly maxChildrenPerParent: number;
|
|
1103
|
+
private readonly budgetBytesValue: number;
|
|
1104
|
+
private total = 0;
|
|
1105
|
+
|
|
1106
|
+
constructor(options: { maxParents?: number; maxChildrenPerParent?: number; budgetBytes?: number } = {}) {
|
|
1107
|
+
this.maxParents = options.maxParents ?? PARENT_CHILDREN_MAX_PARENTS;
|
|
1108
|
+
this.maxChildrenPerParent = options.maxChildrenPerParent ?? PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT;
|
|
1109
|
+
this.budgetBytesValue = options.budgetBytes ?? PARENT_CHILDREN_BUDGET_BYTES;
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
get size(): number {
|
|
1113
|
+
return this.childrenByParent.size;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
get totalBytes(): number {
|
|
1117
|
+
return this.total;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
get budgetBytes(): number {
|
|
1121
|
+
return this.budgetBytesValue;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
add(parentId: string, childId: string): boolean {
|
|
1125
|
+
if (parentId.length === 0 || childId.length === 0) throw new Error("parent_children_empty_id");
|
|
1126
|
+
let children = this.childrenByParent.get(parentId);
|
|
1127
|
+
if (children?.includes(childId)) return true;
|
|
1128
|
+
const parentBytes = children === undefined ? residentStringBytes(parentId) + RECORD_OBJECT_OVERHEAD_BYTES : 0;
|
|
1129
|
+
const childBytes = residentStringBytes(childId) + 8;
|
|
1130
|
+
if (this.total + parentBytes + childBytes > this.budgetBytesValue) return false;
|
|
1131
|
+
if (children === undefined) {
|
|
1132
|
+
if (this.childrenByParent.size >= this.maxParents) return false;
|
|
1133
|
+
children = [];
|
|
1134
|
+
this.childrenByParent.set(parentId, children);
|
|
1135
|
+
}
|
|
1136
|
+
if (children.length >= this.maxChildrenPerParent) return false;
|
|
1137
|
+
children.push(childId);
|
|
1138
|
+
this.total += parentBytes + childBytes;
|
|
1139
|
+
return true;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
get(parentId: string): readonly string[] | undefined {
|
|
1143
|
+
const children = this.childrenByParent.get(parentId);
|
|
1144
|
+
return children ? [...children] : undefined;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
entries(): ParentChildrenIndexEntry[] {
|
|
1148
|
+
const out: ParentChildrenIndexEntry[] = [];
|
|
1149
|
+
for (const [parentId, children] of this.childrenByParent) out.push({ parentId, children: [...children] });
|
|
1150
|
+
return out;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* Bounded labels/pins store. Accounting uses `stored bytes + map overhead`
|
|
1156
|
+
* (resident string formulas + per-entry overhead), rejected when the labels+pins
|
|
1157
|
+
* budget would be exceeded.
|
|
1158
|
+
*/
|
|
1159
|
+
export class BoundedLabelsPinsStore {
|
|
1160
|
+
private readonly labels = new Map<string, string>();
|
|
1161
|
+
private readonly pins = new Map<string, string>();
|
|
1162
|
+
private total = 0;
|
|
1163
|
+
private readonly budget: number;
|
|
1164
|
+
|
|
1165
|
+
constructor(budgetBytes: number = LABELS_PINS_BUDGET_BYTES) {
|
|
1166
|
+
this.budget = budgetBytes;
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
get totalBytes(): number {
|
|
1170
|
+
return this.total;
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
get budgetBytes(): number {
|
|
1174
|
+
return this.budget;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
setLabel(key: string, value: string): boolean {
|
|
1178
|
+
return this.setIn(this.labels, key, value);
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
deleteLabel(key: string): void {
|
|
1182
|
+
const existing = this.labels.get(key);
|
|
1183
|
+
if (existing === undefined) return;
|
|
1184
|
+
this.total -= residentStringBytes(key) + residentStringBytes(existing) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
1185
|
+
this.labels.delete(key);
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
setPin(key: string, value: string): boolean {
|
|
1189
|
+
return this.setIn(this.pins, key, value);
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
getLabel(key: string): string | undefined {
|
|
1193
|
+
return this.labels.get(key);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
getPin(key: string): string | undefined {
|
|
1197
|
+
return this.pins.get(key);
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
labelsEntries(): ReadonlyMap<string, string> {
|
|
1201
|
+
return this.labels;
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
pinsEntries(): ReadonlyMap<string, string> {
|
|
1205
|
+
return this.pins;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
clear(): void {
|
|
1209
|
+
this.labels.clear();
|
|
1210
|
+
this.pins.clear();
|
|
1211
|
+
this.total = 0;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
private setIn(map: Map<string, string>, key: string, value: string): boolean {
|
|
1215
|
+
const cost = residentStringBytes(key) + residentStringBytes(value) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
1216
|
+
const existing = map.get(key);
|
|
1217
|
+
if (existing !== undefined) {
|
|
1218
|
+
const existingCost = residentStringBytes(key) + residentStringBytes(existing) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
1219
|
+
const delta = cost - existingCost;
|
|
1220
|
+
if (this.total + delta > this.budget) return false;
|
|
1221
|
+
this.total += delta;
|
|
1222
|
+
map.set(key, value);
|
|
1223
|
+
return true;
|
|
1224
|
+
}
|
|
1225
|
+
if (this.total + cost > this.budget) return false;
|
|
1226
|
+
this.total += cost;
|
|
1227
|
+
map.set(key, value);
|
|
1228
|
+
return true;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
// ============================================================================
|
|
1233
|
+
// Persistent bounded parent→children artifact (disposable v2 parent index)
|
|
1234
|
+
// ============================================================================
|
|
1235
|
+
|
|
1236
|
+
/**
|
|
1237
|
+
* One parent→child edge plus the child's authoritative cold index fields. The
|
|
1238
|
+
* artifact is disposable derived proof; the `.spill.idx` remains authoritative
|
|
1239
|
+
* for entry offsets and the transcript remains authoritative for entry bytes.
|
|
1240
|
+
*/
|
|
1241
|
+
export interface ParentBucketRecordInput {
|
|
1242
|
+
parentId: string;
|
|
1243
|
+
childId: string;
|
|
1244
|
+
ordinal: number;
|
|
1245
|
+
seq: number;
|
|
1246
|
+
byteOffset: number;
|
|
1247
|
+
byteLength: number;
|
|
1248
|
+
recordDigest: string;
|
|
1249
|
+
entryType?: string;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
/** Committed on-disk state of one parent bucket (exact bytes). */
|
|
1253
|
+
export interface ParentBucketCommit {
|
|
1254
|
+
size: number;
|
|
1255
|
+
digest: string;
|
|
1256
|
+
complete: boolean;
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
/**
|
|
1260
|
+
* Commit-marker binding for the disposable parent artifact. The artifact
|
|
1261
|
+
* covers exactly the entry set of the `.spill.idx` whose digest is bound here;
|
|
1262
|
+
* a binding mismatch fails closed to the authoritative cold scan.
|
|
1263
|
+
*/
|
|
1264
|
+
export interface ParentArtifactCommit {
|
|
1265
|
+
bucketCount: number;
|
|
1266
|
+
indexDigest: string;
|
|
1267
|
+
buckets: ParentBucketCommit[];
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/**
|
|
1271
|
+
* Deterministic hash-partition assignment for a parent id. Bucket files are
|
|
1272
|
+
* append-only journals; a lookup reads exactly one bucket (bounded range read)
|
|
1273
|
+
* and verifies each record's `parentId` to filter hash collisions.
|
|
1274
|
+
*/
|
|
1275
|
+
export function parentBucketForId(parentId: string, bucketCount = PARENT_CHILDREN_BUCKET_COUNT): number {
|
|
1276
|
+
let hash = 0x811c9dc5;
|
|
1277
|
+
for (let index = 0; index < parentId.length; index++) {
|
|
1278
|
+
const code = parentId.charCodeAt(index);
|
|
1279
|
+
hash ^= code & 0xff;
|
|
1280
|
+
hash = Math.imul(hash, 0x01000193);
|
|
1281
|
+
hash ^= code >>> 8;
|
|
1282
|
+
hash = Math.imul(hash, 0x01000193);
|
|
1283
|
+
}
|
|
1284
|
+
return (hash >>> 0) % bucketCount;
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
/** Serialize one parent→child edge as a compact JSON journal line (trailing newline). */
|
|
1288
|
+
export function serializeParentBucketRecord(input: ParentBucketRecordInput): string {
|
|
1289
|
+
return `${JSON.stringify({
|
|
1290
|
+
p: input.parentId,
|
|
1291
|
+
c: input.childId,
|
|
1292
|
+
o: input.ordinal,
|
|
1293
|
+
s: input.seq,
|
|
1294
|
+
b: input.byteOffset,
|
|
1295
|
+
l: input.byteLength,
|
|
1296
|
+
d: input.recordDigest,
|
|
1297
|
+
...(input.entryType !== undefined ? { t: input.entryType } : {}),
|
|
1298
|
+
})}\n`;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** Strictly parse one parent bucket journal line; `undefined` marks the bucket corrupt. */
|
|
1302
|
+
export function parseParentBucketRecord(line: string): ParentBucketRecordInput | undefined {
|
|
1303
|
+
let value: Record<string, unknown>;
|
|
1304
|
+
try {
|
|
1305
|
+
value = JSON.parse(line) as Record<string, unknown>;
|
|
1306
|
+
} catch {
|
|
1307
|
+
return undefined;
|
|
1308
|
+
}
|
|
1309
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
1310
|
+
const { p, c, o, s, b, l, d, t } = value;
|
|
1311
|
+
if (typeof p !== "string" || p.length === 0) return undefined;
|
|
1312
|
+
if (typeof c !== "string" || c.length === 0) return undefined;
|
|
1313
|
+
if (typeof o !== "number" || typeof s !== "number" || typeof b !== "number" || typeof l !== "number")
|
|
1314
|
+
return undefined;
|
|
1315
|
+
if (!Number.isSafeInteger(o) || !Number.isSafeInteger(s) || !Number.isSafeInteger(b) || !Number.isSafeInteger(l))
|
|
1316
|
+
return undefined;
|
|
1317
|
+
if (typeof d !== "string" || d.length !== 64) return undefined;
|
|
1318
|
+
if (t !== undefined && typeof t !== "string") return undefined;
|
|
1319
|
+
return {
|
|
1320
|
+
parentId: p,
|
|
1321
|
+
childId: c,
|
|
1322
|
+
ordinal: o,
|
|
1323
|
+
seq: s,
|
|
1324
|
+
byteOffset: b,
|
|
1325
|
+
byteLength: l,
|
|
1326
|
+
recordDigest: d,
|
|
1327
|
+
...(t !== undefined ? { entryType: t } : {}),
|
|
1328
|
+
};
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
export interface ParentArtifactBuildResult {
|
|
1332
|
+
metadata: ParentArtifactCommit;
|
|
1333
|
+
/** Per-bucket records in physical (transcript) order. */
|
|
1334
|
+
buckets: string[][];
|
|
1335
|
+
/** Parents excluded because a bound (children/budget/parents) was exceeded. */
|
|
1336
|
+
excludedParents: readonly string[];
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
/**
|
|
1340
|
+
* Bounded builder for the persistent parent artifact. Parents are indexed
|
|
1341
|
+
* fully or not at all: a parent whose children would exceed
|
|
1342
|
+
* `PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT`, whose first record would exceed
|
|
1343
|
+
* `PARENT_CHILDREN_BUDGET_BYTES`, or that would exceed
|
|
1344
|
+
* `PARENT_CHILDREN_MAX_PARENTS` is excluded (lookups for it fail closed), and
|
|
1345
|
+
* its buffered records are dropped so a truncated "complete" result can never
|
|
1346
|
+
* be published. Per-parent child order is preserved (insertion order).
|
|
1347
|
+
*/
|
|
1348
|
+
export class BoundedParentArtifactBuilder {
|
|
1349
|
+
private readonly parents = new Map<string, { records: string[]; bytes: number }>();
|
|
1350
|
+
private readonly excluded = new Set<string>();
|
|
1351
|
+
private readonly incompleteBuckets = new Set<number>();
|
|
1352
|
+
private totalBytesValue = 0;
|
|
1353
|
+
private readonly maxParents: number;
|
|
1354
|
+
private readonly maxChildrenPerParent: number;
|
|
1355
|
+
private readonly budgetBytesValue: number;
|
|
1356
|
+
private readonly bucketCount: number;
|
|
1357
|
+
private finished = false;
|
|
1358
|
+
private capacityExhausted = false;
|
|
1359
|
+
|
|
1360
|
+
constructor(
|
|
1361
|
+
options: {
|
|
1362
|
+
maxParents?: number;
|
|
1363
|
+
maxChildrenPerParent?: number;
|
|
1364
|
+
budgetBytes?: number;
|
|
1365
|
+
bucketCount?: number;
|
|
1366
|
+
} = {},
|
|
1367
|
+
) {
|
|
1368
|
+
this.maxParents = options.maxParents ?? PARENT_CHILDREN_MAX_PARENTS;
|
|
1369
|
+
this.maxChildrenPerParent = options.maxChildrenPerParent ?? PARENT_CHILDREN_MAX_CHILDREN_PER_PARENT;
|
|
1370
|
+
this.budgetBytesValue = options.budgetBytes ?? PARENT_CHILDREN_BUDGET_BYTES;
|
|
1371
|
+
this.bucketCount = options.bucketCount ?? PARENT_CHILDREN_BUCKET_COUNT;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
get distinctParents(): number {
|
|
1375
|
+
return this.parents.size;
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
get totalBytes(): number {
|
|
1379
|
+
return this.totalBytesValue;
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
add(record: ParentBucketRecordInput): void {
|
|
1383
|
+
if (this.finished) throw new Error("parent_artifact_builder_finished");
|
|
1384
|
+
if (record.parentId.length === 0 || record.childId.length === 0) throw new Error("parent_artifact_empty_id");
|
|
1385
|
+
if (this.excluded.has(record.parentId)) return;
|
|
1386
|
+
if (this.capacityExhausted && !this.parents.has(record.parentId)) {
|
|
1387
|
+
this.incompleteBuckets.add(parentBucketForId(record.parentId, this.bucketCount));
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
const line = serializeParentBucketRecord(record);
|
|
1391
|
+
const bytes = Buffer.byteLength(line, "utf8");
|
|
1392
|
+
let parent = this.parents.get(record.parentId);
|
|
1393
|
+
if (parent === undefined) {
|
|
1394
|
+
if (this.parents.size >= this.maxParents || this.totalBytesValue + bytes > this.budgetBytesValue) {
|
|
1395
|
+
this.capacityExhausted = true;
|
|
1396
|
+
this.incompleteBuckets.add(parentBucketForId(record.parentId, this.bucketCount));
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
parent = { records: [], bytes: 0 };
|
|
1400
|
+
this.parents.set(record.parentId, parent);
|
|
1401
|
+
}
|
|
1402
|
+
if (parent.records.length >= this.maxChildrenPerParent || this.totalBytesValue + bytes > this.budgetBytesValue) {
|
|
1403
|
+
this.exclude(record.parentId);
|
|
1404
|
+
return;
|
|
1405
|
+
}
|
|
1406
|
+
parent.records.push(line);
|
|
1407
|
+
parent.bytes += bytes;
|
|
1408
|
+
this.totalBytesValue += bytes;
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
private exclude(parentId: string): void {
|
|
1412
|
+
const parent = this.parents.get(parentId);
|
|
1413
|
+
if (parent) {
|
|
1414
|
+
this.totalBytesValue -= parent.bytes;
|
|
1415
|
+
this.parents.delete(parentId);
|
|
1416
|
+
}
|
|
1417
|
+
this.incompleteBuckets.add(parentBucketForId(parentId, this.bucketCount));
|
|
1418
|
+
this.excluded.add(parentId);
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
finish(indexDigest: string): ParentArtifactBuildResult {
|
|
1422
|
+
if (this.finished) throw new Error("parent_artifact_builder_already_finished");
|
|
1423
|
+
this.finished = true;
|
|
1424
|
+
const byBucket: Array<{ records: string[]; bytes: number }> = Array.from({ length: this.bucketCount }, () => ({
|
|
1425
|
+
records: [],
|
|
1426
|
+
bytes: 0,
|
|
1427
|
+
}));
|
|
1428
|
+
for (const [parentId, parent] of this.parents) {
|
|
1429
|
+
const bucket = parentBucketForId(parentId, this.bucketCount);
|
|
1430
|
+
for (const line of parent.records) {
|
|
1431
|
+
byBucket[bucket].records.push(line);
|
|
1432
|
+
byBucket[bucket].bytes += Buffer.byteLength(line, "utf8");
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
const buckets = byBucket.map((bucket, bucketIndex) => {
|
|
1436
|
+
const hash = createHash("sha256");
|
|
1437
|
+
for (const line of bucket.records) hash.update(line, "utf8");
|
|
1438
|
+
return {
|
|
1439
|
+
size: bucket.bytes,
|
|
1440
|
+
digest: hash.digest("hex"),
|
|
1441
|
+
complete: !this.incompleteBuckets.has(bucketIndex),
|
|
1442
|
+
};
|
|
1443
|
+
});
|
|
1444
|
+
const result: ParentArtifactBuildResult = {
|
|
1445
|
+
metadata: { bucketCount: this.bucketCount, indexDigest, buckets },
|
|
1446
|
+
buckets: byBucket.map(bucket => bucket.records),
|
|
1447
|
+
excludedParents: [...this.excluded],
|
|
1448
|
+
};
|
|
1449
|
+
this.parents.clear();
|
|
1450
|
+
this.excluded.clear();
|
|
1451
|
+
this.incompleteBuckets.clear();
|
|
1452
|
+
this.totalBytesValue = 0;
|
|
1453
|
+
return result;
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/** Fixed in-memory cost of the retained parent-artifact runtime state (block-cache charged). */
|
|
1458
|
+
export function parentArtifactRuntimeBytes(bucketCount = PARENT_CHILDREN_BUCKET_COUNT): number {
|
|
1459
|
+
return bucketCount * 96 + 128;
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
// ============================================================================
|
|
1463
|
+
// Persistent bounded dictionary artifact (deterministic hash partitions)
|
|
1464
|
+
// ============================================================================
|
|
1465
|
+
|
|
1466
|
+
/** Fixed partition count for the on-disk dictionary artifact (4 × 4 MiB buffers). */
|
|
1467
|
+
export const DICTIONARY_PARTITION_COUNT = 4;
|
|
1468
|
+
/** Maximum unique duplicate record ids retained as diagnostics in the meta. */
|
|
1469
|
+
export const DICTIONARY_DUPLICATE_DIAGNOSTIC_MAX = 64;
|
|
1470
|
+
/** Fixed capacity of the bounded duplicate-id detector (open addressing). */
|
|
1471
|
+
export const DICTIONARY_ID_SET_CAPACITY = 1_125_001;
|
|
1472
|
+
/** Maximum records the bounded duplicate-id detector can prove unique. */
|
|
1473
|
+
export const DICTIONARY_ID_SET_MAX_RECORDS = 1_000_000;
|
|
1474
|
+
|
|
1475
|
+
/**
|
|
1476
|
+
* One partition journal record. `term` is the lossless record id (never
|
|
1477
|
+
* truncated or hash-substituted); the remaining fields are the same exact
|
|
1478
|
+
* index fields the covered `.spill.idx` records, so a partition is a
|
|
1479
|
+
* self-sufficient hash-partitioned flat index for one id.
|
|
1480
|
+
*/
|
|
1481
|
+
export interface DictionaryPartitionRecord {
|
|
1482
|
+
term: string;
|
|
1483
|
+
dictId: number;
|
|
1484
|
+
ordinal: number;
|
|
1485
|
+
seq: number;
|
|
1486
|
+
byteOffset: number;
|
|
1487
|
+
byteLength: number;
|
|
1488
|
+
recordDigest: string;
|
|
1489
|
+
parentId: string | null;
|
|
1490
|
+
entryType: string;
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
/** Serialize one dictionary partition record as a compact JSON journal line (trailing newline). */
|
|
1494
|
+
export function serializeDictionaryPartitionRecord(input: DictionaryPartitionRecord): string {
|
|
1495
|
+
return `${JSON.stringify({
|
|
1496
|
+
t: input.term,
|
|
1497
|
+
i: input.dictId,
|
|
1498
|
+
o: input.ordinal,
|
|
1499
|
+
s: input.seq,
|
|
1500
|
+
b: input.byteOffset,
|
|
1501
|
+
l: input.byteLength,
|
|
1502
|
+
d: input.recordDigest,
|
|
1503
|
+
p: input.parentId,
|
|
1504
|
+
e: input.entryType,
|
|
1505
|
+
})}\n`;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
/** Strictly parse one dictionary partition journal line; `undefined` marks the partition corrupt. */
|
|
1509
|
+
export function parseDictionaryPartitionRecord(line: string): DictionaryPartitionRecord | undefined {
|
|
1510
|
+
let value: Record<string, unknown>;
|
|
1511
|
+
try {
|
|
1512
|
+
value = JSON.parse(line) as Record<string, unknown>;
|
|
1513
|
+
} catch {
|
|
1514
|
+
return undefined;
|
|
1515
|
+
}
|
|
1516
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
1517
|
+
const { t, i, o, s, b, l, d, p, e } = value;
|
|
1518
|
+
if (typeof t !== "string" || t.length === 0) return undefined;
|
|
1519
|
+
if (typeof i !== "number" || !Number.isSafeInteger(i) || i < 0) return undefined;
|
|
1520
|
+
if (
|
|
1521
|
+
typeof o !== "number" ||
|
|
1522
|
+
typeof s !== "number" ||
|
|
1523
|
+
typeof b !== "number" ||
|
|
1524
|
+
typeof l !== "number" ||
|
|
1525
|
+
!Number.isSafeInteger(o) ||
|
|
1526
|
+
!Number.isSafeInteger(s) ||
|
|
1527
|
+
!Number.isSafeInteger(b) ||
|
|
1528
|
+
!Number.isSafeInteger(l) ||
|
|
1529
|
+
o < 0 ||
|
|
1530
|
+
s < 0 ||
|
|
1531
|
+
b < 0 ||
|
|
1532
|
+
l <= 0
|
|
1533
|
+
)
|
|
1534
|
+
return undefined;
|
|
1535
|
+
if (typeof d !== "string" || !/^[0-9a-f]{64}$/.test(d)) return undefined;
|
|
1536
|
+
if (p !== null && typeof p !== "string") return undefined;
|
|
1537
|
+
if (typeof e !== "string" || e.length === 0) return undefined;
|
|
1538
|
+
return {
|
|
1539
|
+
term: t,
|
|
1540
|
+
dictId: i,
|
|
1541
|
+
ordinal: o,
|
|
1542
|
+
seq: s,
|
|
1543
|
+
byteOffset: b,
|
|
1544
|
+
byteLength: l,
|
|
1545
|
+
recordDigest: d,
|
|
1546
|
+
parentId: p,
|
|
1547
|
+
entryType: e,
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/**
|
|
1552
|
+
* Deterministic hash-partition assignment for a record id (same FNV-1a family
|
|
1553
|
+
* as `parentBucketForId`). The same id always lands in the same partition, so
|
|
1554
|
+
* every occurrence of a duplicate id is confined to one journal.
|
|
1555
|
+
*/
|
|
1556
|
+
export function dictionaryPartitionForId(id: string, partitionCount = DICTIONARY_PARTITION_COUNT): number {
|
|
1557
|
+
let hash = 0x811c9dc5;
|
|
1558
|
+
for (let index = 0; index < id.length; index++) {
|
|
1559
|
+
const code = id.charCodeAt(index);
|
|
1560
|
+
hash ^= code & 0xff;
|
|
1561
|
+
hash = Math.imul(hash, 0x01000193);
|
|
1562
|
+
hash ^= code >>> 8;
|
|
1563
|
+
hash = Math.imul(hash, 0x01000193);
|
|
1564
|
+
}
|
|
1565
|
+
return (hash >>> 0) % partitionCount;
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
/** Committed exact bytes of one on-disk dictionary partition. */
|
|
1569
|
+
export interface DictionaryPartitionCommit {
|
|
1570
|
+
size: number;
|
|
1571
|
+
digest: string;
|
|
1572
|
+
records: number;
|
|
1573
|
+
complete: boolean;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
/**
|
|
1577
|
+
* Commit-marker binding for the disposable dictionary artifact. Every partition
|
|
1578
|
+
* and the meta file are bound by exact size + SHA-256, and the artifact covers
|
|
1579
|
+
* exactly the entry set of the `.spill.idx` whose digest is bound here; a
|
|
1580
|
+
* binding mismatch fails closed to the authoritative cold scan.
|
|
1581
|
+
*/
|
|
1582
|
+
export interface DictionaryArtifactCommit {
|
|
1583
|
+
header: SidecarIndexHeaderV2;
|
|
1584
|
+
/** Exact `.spill.idx` digest the artifact covers. */
|
|
1585
|
+
indexDigest: string;
|
|
1586
|
+
/** Per-partition exact-byte commits, one per partition file. */
|
|
1587
|
+
partitions: DictionaryPartitionCommit[];
|
|
1588
|
+
/** Exact bytes of the `.spill.dict-meta` file (self-referential fields omitted from the payload). */
|
|
1589
|
+
metaSize: number;
|
|
1590
|
+
metaDigest: string;
|
|
1591
|
+
recordCount: number;
|
|
1592
|
+
uniqueTerms: number;
|
|
1593
|
+
/** Combined serialized term bytes (lossless ids). */
|
|
1594
|
+
totalBytes: number;
|
|
1595
|
+
/** Bounded duplicate-id diagnostics; non-empty ⇒ the artifact must never be adopted. */
|
|
1596
|
+
duplicateIds: readonly string[];
|
|
1597
|
+
sidecarIneligible: boolean;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Exact serialized bytes of the meta file. `metaSize`/`metaDigest` are omitted
|
|
1602
|
+
* from the payload (they authenticate the bytes that omit them), so
|
|
1603
|
+
* serialization is deterministic and self-referentiality is impossible.
|
|
1604
|
+
*/
|
|
1605
|
+
export function dictionaryArtifactMetaBytes(commit: Omit<DictionaryArtifactCommit, "metaSize" | "metaDigest">): string {
|
|
1606
|
+
return `${JSON.stringify({
|
|
1607
|
+
header: {
|
|
1608
|
+
version: commit.header.version,
|
|
1609
|
+
sessionId: commit.header.sessionId,
|
|
1610
|
+
sidecarIneligible: commit.header.sidecarIneligible,
|
|
1611
|
+
},
|
|
1612
|
+
indexDigest: commit.indexDigest,
|
|
1613
|
+
partitions: commit.partitions.map(partition => ({
|
|
1614
|
+
size: partition.size,
|
|
1615
|
+
digest: partition.digest,
|
|
1616
|
+
records: partition.records,
|
|
1617
|
+
complete: partition.complete,
|
|
1618
|
+
})),
|
|
1619
|
+
recordCount: commit.recordCount,
|
|
1620
|
+
uniqueTerms: commit.uniqueTerms,
|
|
1621
|
+
totalBytes: commit.totalBytes,
|
|
1622
|
+
duplicateIds: [...commit.duplicateIds],
|
|
1623
|
+
sidecarIneligible: commit.sidecarIneligible,
|
|
1624
|
+
})}\n`;
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
/** Compute the exact meta bytes and the resulting self-authenticating commit. */
|
|
1628
|
+
export function finalizeDictionaryArtifactCommit(commit: Omit<DictionaryArtifactCommit, "metaSize" | "metaDigest">): {
|
|
1629
|
+
bytes: string;
|
|
1630
|
+
commit: DictionaryArtifactCommit;
|
|
1631
|
+
} {
|
|
1632
|
+
const bytes = dictionaryArtifactMetaBytes(commit);
|
|
1633
|
+
const metaDigest = computeLineDigest(Buffer.from(bytes, "utf8"));
|
|
1634
|
+
return { bytes, commit: { ...commit, metaSize: bytes.length, metaDigest } };
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
/**
|
|
1638
|
+
* Strictly parse the exact meta bytes. The digest and size are recomputed over
|
|
1639
|
+
* the supplied exact bytes; any structural, bound, or digest mismatch returns
|
|
1640
|
+
* `undefined` (the artifact is then never trusted).
|
|
1641
|
+
*/
|
|
1642
|
+
export function parseDictionaryArtifactCommit(bytes: string | Uint8Array): DictionaryArtifactCommit | undefined {
|
|
1643
|
+
const text = typeof bytes === "string" ? bytes : new TextDecoder("utf-8").decode(bytes);
|
|
1644
|
+
const exactLength = typeof bytes === "string" ? Buffer.byteLength(bytes, "utf8") : bytes.byteLength;
|
|
1645
|
+
let value: unknown;
|
|
1646
|
+
try {
|
|
1647
|
+
value = JSON.parse(text);
|
|
1648
|
+
} catch {
|
|
1649
|
+
return undefined;
|
|
1650
|
+
}
|
|
1651
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
1652
|
+
const record = value as Record<string, unknown>;
|
|
1653
|
+
const header = record.header;
|
|
1654
|
+
if (header === null || typeof header !== "object" || Array.isArray(header)) return undefined;
|
|
1655
|
+
const headerRecord = header as Record<string, unknown>;
|
|
1656
|
+
if (headerRecord.version !== 2) return undefined;
|
|
1657
|
+
if (typeof headerRecord.sessionId !== "string") return undefined;
|
|
1658
|
+
if (typeof headerRecord.sidecarIneligible !== "boolean") return undefined;
|
|
1659
|
+
if (typeof record.indexDigest !== "string" || !/^[0-9a-f]{64}$/.test(record.indexDigest)) return undefined;
|
|
1660
|
+
if (!Array.isArray(record.partitions) || record.partitions.length !== DICTIONARY_PARTITION_COUNT) return undefined;
|
|
1661
|
+
const partitions: DictionaryPartitionCommit[] = [];
|
|
1662
|
+
for (const candidate of record.partitions) {
|
|
1663
|
+
if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) return undefined;
|
|
1664
|
+
const partition = candidate as Record<string, unknown>;
|
|
1665
|
+
if (!Number.isSafeInteger(partition.size) || (partition.size as number) < 0) return undefined;
|
|
1666
|
+
if (typeof partition.digest !== "string" || !/^[0-9a-f]{64}$/.test(partition.digest)) return undefined;
|
|
1667
|
+
if (!Number.isSafeInteger(partition.records) || (partition.records as number) < 0) return undefined;
|
|
1668
|
+
if (typeof partition.complete !== "boolean") return undefined;
|
|
1669
|
+
partitions.push({
|
|
1670
|
+
size: partition.size as number,
|
|
1671
|
+
digest: partition.digest,
|
|
1672
|
+
records: partition.records as number,
|
|
1673
|
+
complete: partition.complete,
|
|
1674
|
+
});
|
|
1675
|
+
}
|
|
1676
|
+
if (!Number.isSafeInteger(record.recordCount) || (record.recordCount as number) < 0) return undefined;
|
|
1677
|
+
if (!Number.isSafeInteger(record.uniqueTerms) || (record.uniqueTerms as number) < 0) return undefined;
|
|
1678
|
+
if (!Number.isSafeInteger(record.totalBytes) || (record.totalBytes as number) < 0) return undefined;
|
|
1679
|
+
if (!Array.isArray(record.duplicateIds) || record.duplicateIds.some(candidate => typeof candidate !== "string"))
|
|
1680
|
+
return undefined;
|
|
1681
|
+
if (typeof record.sidecarIneligible !== "boolean") return undefined;
|
|
1682
|
+
const expectedDigest = computeLineDigest(
|
|
1683
|
+
typeof bytes === "string" ? Buffer.from(bytes, "utf8") : Buffer.from(bytes),
|
|
1684
|
+
);
|
|
1685
|
+
return {
|
|
1686
|
+
header: {
|
|
1687
|
+
version: 2,
|
|
1688
|
+
sessionId: headerRecord.sessionId,
|
|
1689
|
+
sidecarIneligible: headerRecord.sidecarIneligible,
|
|
1690
|
+
},
|
|
1691
|
+
indexDigest: record.indexDigest,
|
|
1692
|
+
partitions,
|
|
1693
|
+
metaSize: exactLength,
|
|
1694
|
+
metaDigest: expectedDigest,
|
|
1695
|
+
recordCount: record.recordCount as number,
|
|
1696
|
+
uniqueTerms: record.uniqueTerms as number,
|
|
1697
|
+
totalBytes: record.totalBytes as number,
|
|
1698
|
+
duplicateIds: record.duplicateIds as string[],
|
|
1699
|
+
sidecarIneligible: record.sidecarIneligible,
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
/** Bounded duplicate-id oracle for the streaming dictionary build. */
|
|
1704
|
+
export interface DictionaryIdDetector {
|
|
1705
|
+
add(id: string): "added" | "duplicate" | "full";
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
/**
|
|
1709
|
+
* Bounded duplicate-id oracle. Small transcripts stay in a tiny exact set;
|
|
1710
|
+
* larger transcripts promote once to the fixed-capacity 48-bit hash table.
|
|
1711
|
+
* A hash collision can only produce a spurious duplicate (fail closed), never
|
|
1712
|
+
* miss a true duplicate.
|
|
1713
|
+
*/
|
|
1714
|
+
export class BoundedDictionaryIdSet implements DictionaryIdDetector {
|
|
1715
|
+
#small: Set<string> | undefined = new Set();
|
|
1716
|
+
#high: Uint16Array | undefined;
|
|
1717
|
+
#low: Uint32Array | undefined;
|
|
1718
|
+
#size = 0;
|
|
1719
|
+
|
|
1720
|
+
#hash(value: string, seed: number): number {
|
|
1721
|
+
let hash = seed >>> 0;
|
|
1722
|
+
for (let index = 0; index < value.length; index++) {
|
|
1723
|
+
hash ^= value.charCodeAt(index);
|
|
1724
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
1725
|
+
}
|
|
1726
|
+
return hash;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
#hashPair(value: string): [number, number] {
|
|
1730
|
+
let high = this.#hash(value, 0x811c9dc5) & 0xffff;
|
|
1731
|
+
const low = this.#hash(value, 0x9e3779b9);
|
|
1732
|
+
if (high === 0 && low === 0) high = 1;
|
|
1733
|
+
return [high, low];
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
#insertHash(high: number, low: number): "added" | "duplicate" | "full" {
|
|
1737
|
+
const highs = this.#high!;
|
|
1738
|
+
const lows = this.#low!;
|
|
1739
|
+
let slot = low % DICTIONARY_ID_SET_CAPACITY;
|
|
1740
|
+
for (let probes = 0; probes < DICTIONARY_ID_SET_CAPACITY; probes++) {
|
|
1741
|
+
const existingHigh = highs[slot];
|
|
1742
|
+
const existingLow = lows[slot];
|
|
1743
|
+
if (existingHigh === high && existingLow === low) return "duplicate";
|
|
1744
|
+
if (existingHigh === 0 && existingLow === 0) {
|
|
1745
|
+
highs[slot] = high;
|
|
1746
|
+
lows[slot] = low;
|
|
1747
|
+
return "added";
|
|
1748
|
+
}
|
|
1749
|
+
slot++;
|
|
1750
|
+
if (slot === DICTIONARY_ID_SET_CAPACITY) slot = 0;
|
|
1751
|
+
}
|
|
1752
|
+
return "full";
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
#promote(): void {
|
|
1756
|
+
this.#high = new Uint16Array(DICTIONARY_ID_SET_CAPACITY);
|
|
1757
|
+
this.#low = new Uint32Array(DICTIONARY_ID_SET_CAPACITY);
|
|
1758
|
+
for (const value of this.#small!) {
|
|
1759
|
+
const [high, low] = this.#hashPair(value);
|
|
1760
|
+
this.#insertHash(high, low);
|
|
1761
|
+
}
|
|
1762
|
+
this.#small = undefined;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
has(value: string): boolean {
|
|
1766
|
+
if (this.#small) return this.#small.has(value);
|
|
1767
|
+
const [high, low] = this.#hashPair(value);
|
|
1768
|
+
const highs = this.#high!;
|
|
1769
|
+
const lows = this.#low!;
|
|
1770
|
+
let slot = low % DICTIONARY_ID_SET_CAPACITY;
|
|
1771
|
+
for (let probes = 0; probes < DICTIONARY_ID_SET_CAPACITY; probes++) {
|
|
1772
|
+
const existingHigh = highs[slot];
|
|
1773
|
+
const existingLow = lows[slot];
|
|
1774
|
+
if (existingHigh === 0 && existingLow === 0) return false;
|
|
1775
|
+
if (existingHigh === high && existingLow === low) return true;
|
|
1776
|
+
slot++;
|
|
1777
|
+
if (slot === DICTIONARY_ID_SET_CAPACITY) slot = 0;
|
|
1778
|
+
}
|
|
1779
|
+
return true;
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
add(value: string): "added" | "duplicate" | "full" {
|
|
1783
|
+
if (this.#size >= DICTIONARY_ID_SET_MAX_RECORDS) return "full";
|
|
1784
|
+
if (this.#small) {
|
|
1785
|
+
if (this.#small.has(value)) return "duplicate";
|
|
1786
|
+
if (this.#small.size < 4096) {
|
|
1787
|
+
this.#small.add(value);
|
|
1788
|
+
this.#size++;
|
|
1789
|
+
return "added";
|
|
1790
|
+
}
|
|
1791
|
+
this.#promote();
|
|
1792
|
+
}
|
|
1793
|
+
const [high, low] = this.#hashPair(value);
|
|
1794
|
+
const result = this.#insertHash(high, low);
|
|
1795
|
+
if (result === "added") this.#size++;
|
|
1796
|
+
return result;
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/** One record consumed by the streaming dictionary artifact builder. */
|
|
1801
|
+
export interface DictionaryArtifactRecordInput {
|
|
1802
|
+
id: string;
|
|
1803
|
+
ordinal: number;
|
|
1804
|
+
seq: number;
|
|
1805
|
+
byteOffset: number;
|
|
1806
|
+
byteLength: number;
|
|
1807
|
+
recordDigest: string;
|
|
1808
|
+
parentId: string | null;
|
|
1809
|
+
entryType: string;
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
/**
|
|
1813
|
+
* Persistence target for flushed partition lines. The target owns the exact
|
|
1814
|
+
* bytes on disk and their running digests (it must keep per-partition hashes
|
|
1815
|
+
* for append-time rebinding); the builder never retains flushed lines.
|
|
1816
|
+
*/
|
|
1817
|
+
export interface DictionaryArtifactFlushTarget {
|
|
1818
|
+
/** Append `lines` to `partition`'s journal, fsyncing before returning false on failure. */
|
|
1819
|
+
writePartitionLines(partition: number, lines: readonly string[]): boolean;
|
|
1820
|
+
/** Exact-byte commit of one partition as currently persisted. */
|
|
1821
|
+
getPartitionCommit(partition: number): DictionaryPartitionCommit;
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
export type DictionaryArtifactAddResult =
|
|
1825
|
+
| { kind: "ok" }
|
|
1826
|
+
| { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number }
|
|
1827
|
+
| { kind: "flush_failed" };
|
|
1828
|
+
|
|
1829
|
+
export type DictionaryArtifactBuildResult =
|
|
1830
|
+
| { kind: "ok"; commit: DictionaryArtifactCommit; stats: DictionaryBuildStats }
|
|
1831
|
+
| { kind: "budget_exceeded"; peakBytes: number; budgetBytes: number }
|
|
1832
|
+
| { kind: "flush_failed" };
|
|
1833
|
+
|
|
1834
|
+
export interface DictionaryArtifactBuildOptions {
|
|
1835
|
+
peakBudgetBytes?: number;
|
|
1836
|
+
partitionBufferBytes?: number;
|
|
1837
|
+
journalBytes?: number;
|
|
1838
|
+
partitionCount?: number;
|
|
1839
|
+
detector?: DictionaryIdDetector;
|
|
1840
|
+
/** Persistence target for flushed partition lines; required for `add`/`finish`. */
|
|
1841
|
+
target?: DictionaryArtifactFlushTarget;
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
/**
|
|
1845
|
+
* Streaming dictionary artifact builder with a deterministic 20 MiB build peak
|
|
1846
|
+
* (4 × `partitionBufferBytes` partition buffers + one `journalBytes` flush
|
|
1847
|
+
* buffer). Consumes records one at a time, flushes full partition buffers
|
|
1848
|
+
* through the journal to the flush target, and never retains the complete
|
|
1849
|
+
* dictionary object/JSON graph. Record ids are stored losslessly as terms.
|
|
1850
|
+
* Duplicate record ids mark the artifact `sidecarIneligible` (never adopted;
|
|
1851
|
+
* the session falls back to eager) while the build still completes
|
|
1852
|
+
* deterministically.
|
|
1853
|
+
*/
|
|
1854
|
+
export class BoundedDictionaryArtifactBuilder {
|
|
1855
|
+
private readonly buffers: string[][] = [];
|
|
1856
|
+
private readonly bufferBytes: number[] = [];
|
|
1857
|
+
private journal: string[] = [];
|
|
1858
|
+
private journalBytes = 0;
|
|
1859
|
+
private journalPartition: number | undefined;
|
|
1860
|
+
private readonly partitionCountValue: number;
|
|
1861
|
+
private readonly partitionBufferBytesValue: number;
|
|
1862
|
+
private readonly journalBytesValue: number;
|
|
1863
|
+
private readonly peakBudgetValue: number;
|
|
1864
|
+
private readonly detector: DictionaryIdDetector;
|
|
1865
|
+
private readonly target: DictionaryArtifactFlushTarget | undefined;
|
|
1866
|
+
private readonly duplicateIds: string[] = [];
|
|
1867
|
+
private duplicateIdBytes = 0;
|
|
1868
|
+
private uniqueTerms = 0;
|
|
1869
|
+
private totalTermBytes = 0;
|
|
1870
|
+
private recordCount = 0;
|
|
1871
|
+
private peakBytesValue = 0;
|
|
1872
|
+
private sidecarIneligible = false;
|
|
1873
|
+
private flushFailed = false;
|
|
1874
|
+
private finished = false;
|
|
1875
|
+
|
|
1876
|
+
constructor(options: DictionaryArtifactBuildOptions = {}) {
|
|
1877
|
+
this.peakBudgetValue = options.peakBudgetBytes ?? DICTIONARY_BUILD_PEAK_BYTES;
|
|
1878
|
+
this.partitionCountValue = options.partitionCount ?? DICTIONARY_PARTITION_COUNT;
|
|
1879
|
+
this.partitionBufferBytesValue = options.partitionBufferBytes ?? DICTIONARY_PARTITION_BUFFER_BYTES;
|
|
1880
|
+
this.journalBytesValue = options.journalBytes ?? DICTIONARY_BUCKET_JOURNAL_BYTES;
|
|
1881
|
+
this.detector = options.detector ?? new BoundedDictionaryIdSet();
|
|
1882
|
+
this.target = options.target;
|
|
1883
|
+
for (let partition = 0; partition < this.partitionCountValue; partition++) {
|
|
1884
|
+
this.buffers.push([]);
|
|
1885
|
+
this.bufferBytes.push(0);
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
get peakBytes(): number {
|
|
1890
|
+
return this.peakBytesValue;
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
/** Consumes one record. The build peak stays within the 20 MiB buffer model. */
|
|
1894
|
+
add(record: DictionaryArtifactRecordInput): DictionaryArtifactAddResult {
|
|
1895
|
+
if (this.finished) throw new Error("dictionary_artifact_builder_finished");
|
|
1896
|
+
this.recordCount += 1;
|
|
1897
|
+
const partition = dictionaryPartitionForId(record.id, this.partitionCountValue);
|
|
1898
|
+
const detection = this.detector.add(record.id);
|
|
1899
|
+
if (detection === "duplicate") {
|
|
1900
|
+
this.sidecarIneligible = true;
|
|
1901
|
+
if (this.duplicateIds.length < DICTIONARY_DUPLICATE_DIAGNOSTIC_MAX && !this.duplicateIds.includes(record.id)) {
|
|
1902
|
+
this.duplicateIdBytes += residentStringBytes(record.id) + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
1903
|
+
this.duplicateIds.push(record.id);
|
|
1904
|
+
}
|
|
1905
|
+
} else if (detection === "full") {
|
|
1906
|
+
// Uniqueness can no longer be proven with bounded memory; fail closed.
|
|
1907
|
+
this.sidecarIneligible = true;
|
|
1908
|
+
}
|
|
1909
|
+
if (detection === "added") {
|
|
1910
|
+
this.uniqueTerms += 1;
|
|
1911
|
+
this.totalTermBytes += Buffer.byteLength(record.id, "utf8");
|
|
1912
|
+
}
|
|
1913
|
+
const line = serializeDictionaryPartitionRecord({
|
|
1914
|
+
term: record.id,
|
|
1915
|
+
dictId: detection === "added" ? this.uniqueTerms - 1 : -1,
|
|
1916
|
+
ordinal: record.ordinal,
|
|
1917
|
+
seq: record.seq,
|
|
1918
|
+
byteOffset: record.byteOffset,
|
|
1919
|
+
byteLength: record.byteLength,
|
|
1920
|
+
recordDigest: record.recordDigest,
|
|
1921
|
+
parentId: record.parentId,
|
|
1922
|
+
entryType: record.entryType,
|
|
1923
|
+
});
|
|
1924
|
+
const lineBytes = Buffer.byteLength(line, "utf8");
|
|
1925
|
+
this.buffers[partition]!.push(line);
|
|
1926
|
+
this.bufferBytes[partition]! += lineBytes;
|
|
1927
|
+
this.updatePeak();
|
|
1928
|
+
if (this.peakBytesValue > this.peakBudgetValue) {
|
|
1929
|
+
return { kind: "budget_exceeded", peakBytes: this.peakBytesValue, budgetBytes: this.peakBudgetValue };
|
|
1930
|
+
}
|
|
1931
|
+
if (this.bufferBytes[partition]! > this.partitionBufferBytesValue) {
|
|
1932
|
+
if (!this.flushPartition(partition)) return { kind: "flush_failed" };
|
|
1933
|
+
}
|
|
1934
|
+
return { kind: "ok" };
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/** Flush every buffer, read the exact partition commits from the target, and finalize. */
|
|
1938
|
+
finish(sessionId: string, indexDigest: string): DictionaryArtifactBuildResult {
|
|
1939
|
+
if (this.finished) throw new Error("dictionary_artifact_builder_already_finished");
|
|
1940
|
+
this.finished = true;
|
|
1941
|
+
for (let partition = 0; partition < this.partitionCountValue; partition++) {
|
|
1942
|
+
if (this.bufferBytes[partition]! > 0) {
|
|
1943
|
+
if (!this.flushPartition(partition)) return { kind: "flush_failed" };
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
if (this.flushFailed) return { kind: "flush_failed" };
|
|
1947
|
+
if (this.peakBytesValue > this.peakBudgetValue) {
|
|
1948
|
+
return { kind: "budget_exceeded", peakBytes: this.peakBytesValue, budgetBytes: this.peakBudgetValue };
|
|
1949
|
+
}
|
|
1950
|
+
if (!this.target) return { kind: "flush_failed" };
|
|
1951
|
+
const partitions: DictionaryPartitionCommit[] = [];
|
|
1952
|
+
for (let partition = 0; partition < this.partitionCountValue; partition++) {
|
|
1953
|
+
partitions.push(this.target.getPartitionCommit(partition));
|
|
1954
|
+
}
|
|
1955
|
+
const commit = {
|
|
1956
|
+
header: { version: 2 as const, sessionId, sidecarIneligible: this.sidecarIneligible },
|
|
1957
|
+
indexDigest,
|
|
1958
|
+
partitions,
|
|
1959
|
+
metaSize: 0,
|
|
1960
|
+
metaDigest: "",
|
|
1961
|
+
recordCount: this.recordCount,
|
|
1962
|
+
uniqueTerms: this.uniqueTerms,
|
|
1963
|
+
totalBytes: this.totalTermBytes,
|
|
1964
|
+
duplicateIds: [...this.duplicateIds],
|
|
1965
|
+
sidecarIneligible: this.sidecarIneligible,
|
|
1966
|
+
};
|
|
1967
|
+
const stats: DictionaryBuildStats = {
|
|
1968
|
+
totalRecords: this.recordCount,
|
|
1969
|
+
uniqueTerms: this.uniqueTerms,
|
|
1970
|
+
totalBytes: this.totalTermBytes,
|
|
1971
|
+
peakBytes: this.peakBytesValue,
|
|
1972
|
+
duplicateIds: [...this.duplicateIds],
|
|
1973
|
+
sidecarIneligible: this.sidecarIneligible,
|
|
1974
|
+
};
|
|
1975
|
+
this.journal = [];
|
|
1976
|
+
this.journalBytes = 0;
|
|
1977
|
+
this.journalPartition = undefined;
|
|
1978
|
+
for (let partition = 0; partition < this.partitionCountValue; partition++) {
|
|
1979
|
+
this.buffers[partition] = [];
|
|
1980
|
+
this.bufferBytes[partition] = 0;
|
|
1981
|
+
}
|
|
1982
|
+
return { kind: "ok", commit, stats };
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
/** Moves one partition's buffer into the journal (≤ 4 MiB) and flushes it to the target. */
|
|
1986
|
+
private flushPartition(partition: number): boolean {
|
|
1987
|
+
const buffer = this.buffers[partition]!;
|
|
1988
|
+
if (buffer.length === 0) return true;
|
|
1989
|
+
if (this.journalPartition !== undefined) {
|
|
1990
|
+
// The previous journal flush failed or was never drained; fail closed.
|
|
1991
|
+
this.flushFailed = true;
|
|
1992
|
+
return false;
|
|
1993
|
+
}
|
|
1994
|
+
// Take as many leading lines as fit the journal cap (≤ 4 MiB); the rest
|
|
1995
|
+
// stays buffered so a flush batch never exceeds the fixed journal bound.
|
|
1996
|
+
let take = 0;
|
|
1997
|
+
let takeBytes = 0;
|
|
1998
|
+
while (take < buffer.length) {
|
|
1999
|
+
const lineBytes = Buffer.byteLength(buffer[take]!, "utf8");
|
|
2000
|
+
if (takeBytes + lineBytes > this.journalBytesValue) break;
|
|
2001
|
+
takeBytes += lineBytes;
|
|
2002
|
+
take++;
|
|
2003
|
+
}
|
|
2004
|
+
if (take === 0) {
|
|
2005
|
+
// A single line exceeds the journal cap; fail closed.
|
|
2006
|
+
this.flushFailed = true;
|
|
2007
|
+
return false;
|
|
2008
|
+
}
|
|
2009
|
+
const lines = take === buffer.length ? buffer : buffer.slice(0, take);
|
|
2010
|
+
this.buffers[partition] = take === buffer.length ? [] : buffer.slice(take);
|
|
2011
|
+
this.bufferBytes[partition]! -= takeBytes;
|
|
2012
|
+
this.journal = lines;
|
|
2013
|
+
this.journalBytes = takeBytes;
|
|
2014
|
+
this.journalPartition = partition;
|
|
2015
|
+
this.updatePeak();
|
|
2016
|
+
const ok = this.flushJournal();
|
|
2017
|
+
this.updatePeak();
|
|
2018
|
+
return ok;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
private flushJournal(): boolean {
|
|
2022
|
+
if (this.journal.length === 0) return true;
|
|
2023
|
+
const partition = this.journalPartition;
|
|
2024
|
+
if (partition === undefined) return true;
|
|
2025
|
+
const lines = this.journal;
|
|
2026
|
+
const bytes = this.journalBytes;
|
|
2027
|
+
this.journal = [];
|
|
2028
|
+
this.journalBytes = 0;
|
|
2029
|
+
this.journalPartition = undefined;
|
|
2030
|
+
const target = this.target;
|
|
2031
|
+
if (!target) {
|
|
2032
|
+
this.flushFailed = true;
|
|
2033
|
+
return false;
|
|
2034
|
+
}
|
|
2035
|
+
const ok = target.writePartitionLines(partition, lines);
|
|
2036
|
+
if (!ok) this.flushFailed = true;
|
|
2037
|
+
if (bytes > this.peakBytesValue) this.peakBytesValue = bytes;
|
|
2038
|
+
return ok;
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
private updatePeak(): void {
|
|
2042
|
+
// Actual buffered content only: at most one partition buffer is being
|
|
2043
|
+
// filled (~4 MiB), the others hold flush remainders, and the journal is
|
|
2044
|
+
// capped at its fixed size — so the peak never exceeds the 20 MiB model
|
|
2045
|
+
// (4 × 4 MiB partition buffers + 4 MiB bucket flush).
|
|
2046
|
+
const current =
|
|
2047
|
+
this.bufferBytes.reduce((total, bytes) => total + bytes, 0) + this.journalBytes + this.duplicateIdBytes;
|
|
2048
|
+
if (current > this.peakBytesValue) this.peakBytesValue = current;
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
/** Fixed in-memory cost of the retained dictionary-artifact runtime state (block-cache charged). */
|
|
2053
|
+
export function dictionaryArtifactRuntimeBytes(partitionCount = DICTIONARY_PARTITION_COUNT): number {
|
|
2054
|
+
return partitionCount * 96 + 128;
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
// ============================================================================
|
|
2058
|
+
// Persistent metadata-delta artifact (demoted reducer/provider values)
|
|
2059
|
+
// ============================================================================
|
|
2060
|
+
|
|
2061
|
+
/**
|
|
2062
|
+
* One demoted provider-state value descriptor: exact ordinal, kind, byte
|
|
2063
|
+
* location, and digest inside the `.spill.metadata-delta` section. The value
|
|
2064
|
+
* bytes are persisted before demotion, so a descriptor always points at
|
|
2065
|
+
* durable authenticated bytes.
|
|
2066
|
+
*/
|
|
2067
|
+
export interface MetadataDeltaValue {
|
|
2068
|
+
/** Provider-state key (`providerStateEntryKey` semantics: type or `type:role`). */
|
|
2069
|
+
key: string;
|
|
2070
|
+
/** Entry type of the demoted value. */
|
|
2071
|
+
kind: string;
|
|
2072
|
+
/** Exact transcript ordinal of the demoted entry. */
|
|
2073
|
+
ordinal: number;
|
|
2074
|
+
/** Position in the build's ordered provider list (reopen merge slot). */
|
|
2075
|
+
position: number;
|
|
2076
|
+
offset: number;
|
|
2077
|
+
length: number;
|
|
2078
|
+
sha256: string;
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/** Commit-marker binding for the disposable metadata-delta section. */
|
|
2082
|
+
export interface MetadataDeltaArtifactCommit {
|
|
2083
|
+
/** Exact `.spill.idx` digest the sidecar set was built with. */
|
|
2084
|
+
indexDigest: string;
|
|
2085
|
+
/** Exact bytes of the `.spill.metadata-delta` file. */
|
|
2086
|
+
size: number;
|
|
2087
|
+
sha256: string;
|
|
2088
|
+
values: readonly MetadataDeltaValue[];
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
/** Strict descriptor validation (any ambiguity fails closed). */
|
|
2092
|
+
export function isValidMetadataDeltaValue(value: unknown): value is MetadataDeltaValue {
|
|
2093
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2094
|
+
const record = value as Record<string, unknown>;
|
|
2095
|
+
return (
|
|
2096
|
+
typeof record.key === "string" &&
|
|
2097
|
+
record.key.length > 0 &&
|
|
2098
|
+
typeof record.kind === "string" &&
|
|
2099
|
+
record.kind.length > 0 &&
|
|
2100
|
+
Number.isSafeInteger(record.ordinal) &&
|
|
2101
|
+
(record.ordinal as number) >= 0 &&
|
|
2102
|
+
Number.isSafeInteger(record.position) &&
|
|
2103
|
+
(record.position as number) >= 0 &&
|
|
2104
|
+
Number.isSafeInteger(record.offset) &&
|
|
2105
|
+
(record.offset as number) >= 0 &&
|
|
2106
|
+
Number.isSafeInteger(record.length) &&
|
|
2107
|
+
(record.length as number) > 0 &&
|
|
2108
|
+
typeof record.sha256 === "string" &&
|
|
2109
|
+
/^[0-9a-f]{64}$/.test(record.sha256)
|
|
2110
|
+
);
|
|
2111
|
+
}
|
|
2112
|
+
|
|
2113
|
+
/** Strict commit-binding validation: bounded values, unique keys/positions, in-range offsets. */
|
|
2114
|
+
export function isValidMetadataDeltaCommit(value: unknown): value is MetadataDeltaArtifactCommit {
|
|
2115
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2116
|
+
const record = value as Record<string, unknown>;
|
|
2117
|
+
if (typeof record.indexDigest !== "string" || !/^[0-9a-f]{64}$/.test(record.indexDigest)) return false;
|
|
2118
|
+
if (!Number.isSafeInteger(record.size) || (record.size as number) < 0) return false;
|
|
2119
|
+
if (typeof record.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(record.sha256)) return false;
|
|
2120
|
+
if (!Array.isArray(record.values) || record.values.length > 256) return false;
|
|
2121
|
+
if (!record.values.every(isValidMetadataDeltaValue)) return false;
|
|
2122
|
+
const keys = new Set<string>();
|
|
2123
|
+
const positions = new Set<number>();
|
|
2124
|
+
for (const value of record.values) {
|
|
2125
|
+
// Each value is independently authenticated by its exact byte digest; the
|
|
2126
|
+
// marker may legitimately reorder re-demoted values relative to their
|
|
2127
|
+
// byte offsets, so only uniqueness and range bounds are enforced here.
|
|
2128
|
+
if (keys.has(value.key)) return false;
|
|
2129
|
+
if (positions.has(value.position)) return false;
|
|
2130
|
+
keys.add(value.key);
|
|
2131
|
+
positions.add(value.position);
|
|
2132
|
+
if (value.offset + value.length > (record.size as number)) return false;
|
|
2133
|
+
}
|
|
2134
|
+
return true;
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
/** Compute the descriptor for one value whose exact line bytes are about to be persisted. */
|
|
2138
|
+
export function metadataDeltaValueDescriptor(input: {
|
|
2139
|
+
key: string;
|
|
2140
|
+
kind: string;
|
|
2141
|
+
ordinal: number;
|
|
2142
|
+
position: number;
|
|
2143
|
+
offset: number;
|
|
2144
|
+
lineBytes: Uint8Array;
|
|
2145
|
+
}): MetadataDeltaValue {
|
|
2146
|
+
return {
|
|
2147
|
+
key: input.key,
|
|
2148
|
+
kind: input.kind,
|
|
2149
|
+
ordinal: input.ordinal,
|
|
2150
|
+
position: input.position,
|
|
2151
|
+
offset: input.offset,
|
|
2152
|
+
length: input.lineBytes.byteLength,
|
|
2153
|
+
sha256: computeLineDigest(input.lineBytes),
|
|
2154
|
+
};
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
/** Resident bytes of one retained metadata-delta descriptor (key + 24 B descriptor + object). */
|
|
2158
|
+
export function metadataDeltaDescriptorResidentBytes(key: string): number {
|
|
2159
|
+
return residentStringBytes(key) + DESCRIPTOR_BYTES + RECORD_OBJECT_OVERHEAD_BYTES;
|
|
2160
|
+
}
|