lossless-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/LICENSE +13 -0
- package/README.md +50 -0
- package/dist/index.d.mts +2019 -0
- package/dist/index.mjs +5179 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +46 -0
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,2019 @@
|
|
|
1
|
+
import { Static, TSchema, Type } from "typebox";
|
|
2
|
+
//#region src/assembly.d.ts
|
|
3
|
+
export interface AssemblyMsg {
|
|
4
|
+
role: string;
|
|
5
|
+
content?: unknown;
|
|
6
|
+
/** AgentMessage UserMessage compatibility (Pi requires it on outgoing messages). */
|
|
7
|
+
timestamp?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface AlignedEntry {
|
|
10
|
+
entryId: string;
|
|
11
|
+
role: string;
|
|
12
|
+
text: string;
|
|
13
|
+
/** Raw content blocks when `text` is not a faithful rendering of them
|
|
14
|
+
* (`payloadOf`). Carried so ingest stores the same bytes at every call site. */
|
|
15
|
+
payload?: string;
|
|
16
|
+
fileHint?: {
|
|
17
|
+
path: string;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare const SYNTHETIC_MARKER = "[LCM session memory";
|
|
21
|
+
export declare function tailKeyOf(m: AssemblyMsg | undefined): string;
|
|
22
|
+
/** Characters the estimator counts for a context: the same rendering `cutIndexFor`
|
|
23
|
+
* measures, so a calibration describes the quantity every threshold here is
|
|
24
|
+
* compared against. */
|
|
25
|
+
export declare function contextChars(messages: readonly AssemblyMsg[]): number;
|
|
26
|
+
export declare function cutIndexFor(messages: readonly AssemblyMsg[], keepRecentTokens: number): number | null;
|
|
27
|
+
export interface SyntheticMessage {
|
|
28
|
+
role: "user";
|
|
29
|
+
content: string;
|
|
30
|
+
timestamp: number;
|
|
31
|
+
}
|
|
32
|
+
export type SummaryTier = "rich" | "terse" | "stub";
|
|
33
|
+
export interface RenderableSummary {
|
|
34
|
+
id: number;
|
|
35
|
+
depth: number;
|
|
36
|
+
firstEntryId: string;
|
|
37
|
+
lastEntryId: string;
|
|
38
|
+
text: string;
|
|
39
|
+
thoroughText?: string;
|
|
40
|
+
tier: SummaryTier;
|
|
41
|
+
}
|
|
42
|
+
export declare const STUB_CHARS = 200;
|
|
43
|
+
export type PlainSummary = Omit<RenderableSummary, "tier" | "thoroughText">;
|
|
44
|
+
export declare function renderSummaries(summaries: readonly RenderableSummary[]): string;
|
|
45
|
+
export declare function renderFrontier(nodes: readonly PlainSummary[]): string;
|
|
46
|
+
export declare function buildSynthetic(summaries: readonly RenderableSummary[]): SyntheticMessage;
|
|
47
|
+
//#endregion
|
|
48
|
+
//#region src/commit-policy.d.ts
|
|
49
|
+
export interface AppliedSummary {
|
|
50
|
+
id: number;
|
|
51
|
+
depth: number;
|
|
52
|
+
text: string;
|
|
53
|
+
thoroughText?: string;
|
|
54
|
+
firstEntryId: string;
|
|
55
|
+
lastEntryId: string;
|
|
56
|
+
tier: SummaryTier;
|
|
57
|
+
}
|
|
58
|
+
export type AssemblyAction = "quiet" | "consider-apply" | "stable" | "recommit";
|
|
59
|
+
export interface CommitState {
|
|
60
|
+
cutCount: number;
|
|
61
|
+
synthetic: SyntheticMessage;
|
|
62
|
+
applied: AppliedSummary[];
|
|
63
|
+
summaryId: number;
|
|
64
|
+
appliedAtOccupancy: number;
|
|
65
|
+
tailKey: string;
|
|
66
|
+
}
|
|
67
|
+
export declare const DEFAULT_SWAP_RATIO = 0.7;
|
|
68
|
+
export declare const DEFAULT_RECUT_RATIO = 0.85;
|
|
69
|
+
export declare const WINDOW_SAFETY_CEILING = 0.9;
|
|
70
|
+
/** Ceiling for the resolved recut fraction: a line above 1.0 is unreachable, so
|
|
71
|
+
* the pin never re-cuts and the raw tail grows past the window while "stable"
|
|
72
|
+
* is reported forever. Strictly above WINDOW_SAFETY_CEILING + RECUT_ABOVE_SWAP_MARGIN. */
|
|
73
|
+
export declare const RECUT_CEILING = 0.95;
|
|
74
|
+
export declare const REZONE_MARGIN = 1.18;
|
|
75
|
+
export interface ThresholdConfig {
|
|
76
|
+
swapAtRatio?: number;
|
|
77
|
+
swapAtTokens?: number;
|
|
78
|
+
recutAtRatio?: number;
|
|
79
|
+
recutAtTokens?: number;
|
|
80
|
+
smartZone?: number;
|
|
81
|
+
}
|
|
82
|
+
export type ZoneThresholds = ThresholdConfig;
|
|
83
|
+
export interface ResolvedThresholds {
|
|
84
|
+
swap: number;
|
|
85
|
+
recut: number;
|
|
86
|
+
clamped: boolean;
|
|
87
|
+
}
|
|
88
|
+
export declare function mergeZone(global: ThresholdConfig, zone?: ZoneThresholds): ThresholdConfig;
|
|
89
|
+
export declare function thresholdConflicts(cfg: ThresholdConfig, scope?: string): string[];
|
|
90
|
+
export declare function effectiveThresholds(cfg: ThresholdConfig, contextWindow: number): ResolvedThresholds;
|
|
91
|
+
export declare function nextAction(state: CommitState | null, occupancy: number, swapRatio: number, recutRatio: number): AssemblyAction;
|
|
92
|
+
export declare function cutMayReplace(previous: CommitState | null, newCutCount: number): boolean;
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/metrics.d.ts
|
|
95
|
+
export declare function metricsPath(): string;
|
|
96
|
+
/** The metrics log, oldest generation first, so a window longer than one
|
|
97
|
+
* generation still reads as one history. */
|
|
98
|
+
export declare function metricsLogFiles(): string[];
|
|
99
|
+
export declare function rotateIfNeeded(path: string, maxBytes: number): void;
|
|
100
|
+
export declare function pruneGenerations(path: string, keep: number): void;
|
|
101
|
+
export declare function appendMetric(record: MetricRecord): void;
|
|
102
|
+
export interface UsageLike {
|
|
103
|
+
input?: unknown;
|
|
104
|
+
output?: unknown;
|
|
105
|
+
cacheRead?: unknown;
|
|
106
|
+
cacheWrite?: unknown;
|
|
107
|
+
cost?: {
|
|
108
|
+
input?: unknown;
|
|
109
|
+
output?: unknown;
|
|
110
|
+
total?: unknown;
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
interface MetricBase {
|
|
114
|
+
/** Session tag. Absent on rows written outside a session (`gc`, `import`). */
|
|
115
|
+
session?: string;
|
|
116
|
+
}
|
|
117
|
+
export type MetricSpan = [string, string];
|
|
118
|
+
export type SummaryStage = "leaf" | "condensed";
|
|
119
|
+
export type CompactionStage = "blocking" | "async" | "direct";
|
|
120
|
+
export type SummaryLevel = 1 | 2 | 3;
|
|
121
|
+
/** One model's turn in a summarizer chain, in the order it was tried. */
|
|
122
|
+
type SummarizerAttempt = {
|
|
123
|
+
model: string;
|
|
124
|
+
outcome: "ok" | "empty";
|
|
125
|
+
} | {
|
|
126
|
+
model: string;
|
|
127
|
+
outcome: "threw";
|
|
128
|
+
error: string;
|
|
129
|
+
};
|
|
130
|
+
export type UsageCounters = {
|
|
131
|
+
input: number | null;
|
|
132
|
+
output: number | null;
|
|
133
|
+
cacheRead: number | null;
|
|
134
|
+
cacheWrite: number | null;
|
|
135
|
+
costInput: number | null;
|
|
136
|
+
costOutput: number | null;
|
|
137
|
+
costTotal: number | null;
|
|
138
|
+
};
|
|
139
|
+
/** The counters to put on a row, or `undefined` when the provider billed
|
|
140
|
+
* nothing (an aborted call, or a provider that reports no usage). */
|
|
141
|
+
export declare function billedCounters(usage: UsageLike | undefined): UsageCounters | undefined;
|
|
142
|
+
/** One metrics row per measured assistant message: no numeric counter at all
|
|
143
|
+
* (aborted turn, legacy row) yields `undefined`, so it does not count as a turn
|
|
144
|
+
* in the report's cache-economics averages. */
|
|
145
|
+
export declare function usageRecord(usage: UsageLike | undefined): UsageRow | undefined;
|
|
146
|
+
export type SummarizerStage = "blocking" | "async";
|
|
147
|
+
/** Why a summarizer call failed, as far as the provider's own text says. A
|
|
148
|
+
* closed set, and `unknown` is a real answer: a classification that guesses is
|
|
149
|
+
* worse than a bucket that says it could not tell, and the raw message stays on
|
|
150
|
+
* the row either way. `capped` is not a provider failure: it is the pass giving
|
|
151
|
+
* up after too many of them, and the category beside it says which one. */
|
|
152
|
+
export type SummarizerFailureCategory = "auth" | "rate-limit" | "timeout" | "unavailable" | "length" | "empty" | "capped" | "unknown";
|
|
153
|
+
/** `ok` answered with usable text, `empty` answered with none, `failure` was rejected. */
|
|
154
|
+
export type SummarizerOutcome = "ok" | "empty" | "failure";
|
|
155
|
+
export interface SummarizerUsageOptions {
|
|
156
|
+
/** The resolved model key, as `modelKey` renders it. */
|
|
157
|
+
model: string;
|
|
158
|
+
stage?: SummarizerStage;
|
|
159
|
+
outcome: SummarizerOutcome;
|
|
160
|
+
/** The provider's own stop word, so `length` and `error` are told apart. */
|
|
161
|
+
stopReason?: string;
|
|
162
|
+
session?: string;
|
|
163
|
+
}
|
|
164
|
+
export type SummarizerUsageRow = UsageCounters & {
|
|
165
|
+
event: "lcm";
|
|
166
|
+
kind: "summarizer-usage";
|
|
167
|
+
model: string;
|
|
168
|
+
stage?: SummarizerStage;
|
|
169
|
+
outcome: SummarizerOutcome;
|
|
170
|
+
stopReason?: string;
|
|
171
|
+
session?: string;
|
|
172
|
+
};
|
|
173
|
+
/** One provider call the summarizer made. `usage` rows cannot carry it, since
|
|
174
|
+
* those come from Pi's `message_end` for the session's own messages while the
|
|
175
|
+
* summarizer talks to the model registry directly. */
|
|
176
|
+
export declare function summarizerUsageRecord(usage: UsageLike | undefined, opts: SummarizerUsageOptions): SummarizerUsageRow | undefined;
|
|
177
|
+
export type UsageRow = MetricBase & UsageCounters & {
|
|
178
|
+
event: "usage";
|
|
179
|
+
};
|
|
180
|
+
interface LcmRowBase extends MetricBase {
|
|
181
|
+
event: "lcm";
|
|
182
|
+
}
|
|
183
|
+
export type ContextDecisionRow = LcmRowBase & {
|
|
184
|
+
kind: "context-decision";
|
|
185
|
+
action: string;
|
|
186
|
+
occupancy: number;
|
|
187
|
+
tokens: number;
|
|
188
|
+
window: number;
|
|
189
|
+
soft: number;
|
|
190
|
+
commit: number;
|
|
191
|
+
zone?: string;
|
|
192
|
+
committed: boolean;
|
|
193
|
+
/** Present when `tokens` is the estimator's count rather than Pi's: the
|
|
194
|
+
* reported figure described a context other than the one on hand. */
|
|
195
|
+
tokensEstimated?: true;
|
|
196
|
+
};
|
|
197
|
+
export type ProjectionOverBudgetRow = LcmRowBase & {
|
|
198
|
+
kind: "projection-over-budget";
|
|
199
|
+
nodes: number;
|
|
200
|
+
renderedTokens: number;
|
|
201
|
+
budgetTokens: number;
|
|
202
|
+
};
|
|
203
|
+
export type SwapAppliedRow = LcmRowBase & {
|
|
204
|
+
kind: "swap-applied";
|
|
205
|
+
summaryId: number;
|
|
206
|
+
cutCount: number;
|
|
207
|
+
distinctBoundary: boolean;
|
|
208
|
+
appliedAtOccupancy: number;
|
|
209
|
+
nodes: number;
|
|
210
|
+
stubbed: number;
|
|
211
|
+
renderedTokens: number;
|
|
212
|
+
budgetTokens: number;
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* A swap the policy did not commit. `tokens` is the occupancy that triggered
|
|
216
|
+
* the bail; the kick-backoff writer names the entry it declined to kick instead.
|
|
217
|
+
*/
|
|
218
|
+
export type SwapBlockedRow = LcmRowBase & {
|
|
219
|
+
kind: "swap-blocked";
|
|
220
|
+
reason: string;
|
|
221
|
+
occupancy: number;
|
|
222
|
+
tokens?: number;
|
|
223
|
+
entryId?: string;
|
|
224
|
+
};
|
|
225
|
+
export type PiCompactionRow = LcmRowBase & {
|
|
226
|
+
kind: "pi-compaction";
|
|
227
|
+
entryId: string;
|
|
228
|
+
tokens?: number;
|
|
229
|
+
window?: number;
|
|
230
|
+
};
|
|
231
|
+
/** One calibration sample against Pi's own count: the characters this build
|
|
232
|
+
* counted for the growth of the context, the tokens Pi reported for it, and the
|
|
233
|
+
* ratio the samples now support. A null ratio is the answer "cannot say", which is
|
|
234
|
+
* why the row carries the sample beside it. */
|
|
235
|
+
export type EstimateCalibratedRow = LcmRowBase & {
|
|
236
|
+
kind: "estimate-calibrated";
|
|
237
|
+
chars: number;
|
|
238
|
+
tokens: number;
|
|
239
|
+
charsPerToken: number | null;
|
|
240
|
+
samples: number;
|
|
241
|
+
};
|
|
242
|
+
/** A provider stated the real context window in an overflow error, which is the
|
|
243
|
+
* one window number in this system that is not a belief. */
|
|
244
|
+
export type WindowCorrectedRow = LcmRowBase & {
|
|
245
|
+
kind: "window-corrected";
|
|
246
|
+
model: string;
|
|
247
|
+
window: number;
|
|
248
|
+
source: string;
|
|
249
|
+
};
|
|
250
|
+
/** A refused request proved the window in use was too big without naming the
|
|
251
|
+
* real one: the window for the next turn is floored at what the request carried,
|
|
252
|
+
* and `window` is the stored correction when one exists. */
|
|
253
|
+
export type WindowFloorRow = LcmRowBase & {
|
|
254
|
+
kind: "window-floor";
|
|
255
|
+
tokens: number;
|
|
256
|
+
window: number | null;
|
|
257
|
+
};
|
|
258
|
+
export type ConfigProblemRow = LcmRowBase & {
|
|
259
|
+
kind: "config-problem";
|
|
260
|
+
problems: string[];
|
|
261
|
+
};
|
|
262
|
+
export type SummaryInputClippedRow = LcmRowBase & {
|
|
263
|
+
kind: "summary-input-clipped";
|
|
264
|
+
entryId: string;
|
|
265
|
+
keptChars: number;
|
|
266
|
+
droppedChars: number;
|
|
267
|
+
};
|
|
268
|
+
export type SummaryFallbackStoredRow = LcmRowBase & {
|
|
269
|
+
kind: "summary-fallback-stored";
|
|
270
|
+
stage: SummaryStage;
|
|
271
|
+
span: MetricSpan;
|
|
272
|
+
sourceChars: number;
|
|
273
|
+
};
|
|
274
|
+
export type SummaryStoredRow = LcmRowBase & {
|
|
275
|
+
kind: "summary-stored";
|
|
276
|
+
stage: SummaryStage;
|
|
277
|
+
level: SummaryLevel;
|
|
278
|
+
depth: number;
|
|
279
|
+
tokens: number;
|
|
280
|
+
summaryId: number;
|
|
281
|
+
span: MetricSpan;
|
|
282
|
+
};
|
|
283
|
+
export type SummarySkippedCoveredRow = LcmRowBase & {
|
|
284
|
+
kind: "summary-skipped-covered";
|
|
285
|
+
stage: SummaryStage;
|
|
286
|
+
span: MetricSpan;
|
|
287
|
+
messages: number;
|
|
288
|
+
};
|
|
289
|
+
export type SummarySkippedExistingRow = LcmRowBase & {
|
|
290
|
+
kind: "summary-skipped-existing";
|
|
291
|
+
stage: SummaryStage;
|
|
292
|
+
summaryId: number;
|
|
293
|
+
depth: number;
|
|
294
|
+
span: MetricSpan;
|
|
295
|
+
attemptedTokens: number;
|
|
296
|
+
existingTokens: number;
|
|
297
|
+
};
|
|
298
|
+
export type SummaryOversizedRow = LcmRowBase & {
|
|
299
|
+
kind: "summary-oversized";
|
|
300
|
+
stage: SummaryStage;
|
|
301
|
+
depth: number;
|
|
302
|
+
tokens: number;
|
|
303
|
+
targetTokens: number;
|
|
304
|
+
sourceTokens: number;
|
|
305
|
+
retried: boolean;
|
|
306
|
+
level: SummaryLevel;
|
|
307
|
+
span: MetricSpan;
|
|
308
|
+
};
|
|
309
|
+
export type SummaryReplacedRow = LcmRowBase & {
|
|
310
|
+
kind: "summary-replaced";
|
|
311
|
+
stage: SummaryStage;
|
|
312
|
+
summaryId: number;
|
|
313
|
+
beforeTokens: number;
|
|
314
|
+
afterTokens: number;
|
|
315
|
+
};
|
|
316
|
+
export type SummaryFinalTruncatedRow = LcmRowBase & {
|
|
317
|
+
kind: "summary-final-truncated";
|
|
318
|
+
stage: CompactionStage;
|
|
319
|
+
span: MetricSpan;
|
|
320
|
+
beforeTokens: number;
|
|
321
|
+
afterTokens: number;
|
|
322
|
+
};
|
|
323
|
+
export type CondensationStalledRow = LcmRowBase & {
|
|
324
|
+
kind: "condensation-stalled";
|
|
325
|
+
reason: string;
|
|
326
|
+
nodes: number;
|
|
327
|
+
minDepth: number;
|
|
328
|
+
frontierTokens: number;
|
|
329
|
+
targetTokens: number;
|
|
330
|
+
cap: number;
|
|
331
|
+
};
|
|
332
|
+
export type CondensationReducedRow = LcmRowBase & {
|
|
333
|
+
kind: "condensation-reduced";
|
|
334
|
+
summaryId: number;
|
|
335
|
+
depth: number;
|
|
336
|
+
level: SummaryLevel;
|
|
337
|
+
beforeTokens: number;
|
|
338
|
+
afterTokens: number;
|
|
339
|
+
targetTokens: number;
|
|
340
|
+
frontierTokens: number;
|
|
341
|
+
};
|
|
342
|
+
export type CompactionCompleteRow = LcmRowBase & {
|
|
343
|
+
kind: "compaction-complete";
|
|
344
|
+
leaves: number;
|
|
345
|
+
condensed: number;
|
|
346
|
+
ingested: number;
|
|
347
|
+
span: MetricSpan;
|
|
348
|
+
};
|
|
349
|
+
/** A summarizer call that answered with nothing, or that threw. An empty answer
|
|
350
|
+
* is its own reason, so only a throw needs the category that says why. */
|
|
351
|
+
export type SummarizerErrorRow = LcmRowBase & {
|
|
352
|
+
kind: "summarizer-error";
|
|
353
|
+
level: SummaryLevel;
|
|
354
|
+
sourceChars: number;
|
|
355
|
+
} & ({
|
|
356
|
+
reason: "empty-response";
|
|
357
|
+
maxTokens: number;
|
|
358
|
+
} | {
|
|
359
|
+
reason: "threw";
|
|
360
|
+
error: string;
|
|
361
|
+
category: SummarizerFailureCategory;
|
|
362
|
+
});
|
|
363
|
+
/** A pass that stopped calling the model and finished its span with the
|
|
364
|
+
* mechanical truncate. One row per completer, not per refused call: a refusal
|
|
365
|
+
* never reaches a provider, so it is not a `summarizer-error`, and the forty
|
|
366
|
+
* calls it stands for are one fact. */
|
|
367
|
+
export type SummarizerCappedRow = LcmRowBase & {
|
|
368
|
+
kind: "summarizer-capped";
|
|
369
|
+
stage?: SummarizerStage;
|
|
370
|
+
fingerprint: string;
|
|
371
|
+
/** Failed walks this completer paid for before it gave up; 0 when the span
|
|
372
|
+
* was already remembered or the refusal was a budget's. */
|
|
373
|
+
failedWalks: number;
|
|
374
|
+
category: SummarizerFailureCategory;
|
|
375
|
+
source: "pass" | "span-memory" | "budget";
|
|
376
|
+
/** Which budget refused: one request's estimate, or the pass's accumulator. */
|
|
377
|
+
reason?: "call" | "pass";
|
|
378
|
+
spentTokens?: number;
|
|
379
|
+
budgetTokens?: number;
|
|
380
|
+
span: MetricSpan;
|
|
381
|
+
};
|
|
382
|
+
export type SummarizerFallbackRow = LcmRowBase & {
|
|
383
|
+
kind: "summarizer-fallback";
|
|
384
|
+
attempts: SummarizerAttempt[];
|
|
385
|
+
used: string | null;
|
|
386
|
+
};
|
|
387
|
+
/** A configured model key that did not resolve, so a substitution is billing. */
|
|
388
|
+
export type SummarizerModelRow = LcmRowBase & {
|
|
389
|
+
kind: "summarizer-model";
|
|
390
|
+
requested: string[];
|
|
391
|
+
chain: string[];
|
|
392
|
+
unresolved: string[];
|
|
393
|
+
};
|
|
394
|
+
/**
|
|
395
|
+
* A compaction failed. The two writers are told apart by stage: the `gc` path
|
|
396
|
+
* has a db path and no session, the hook path has a session and one of the two
|
|
397
|
+
* span handles.
|
|
398
|
+
*/
|
|
399
|
+
export type CompactionErrorRow = LcmRowBase & {
|
|
400
|
+
kind: "compaction-error";
|
|
401
|
+
error: string;
|
|
402
|
+
} & ({
|
|
403
|
+
stage: "gc" | "migrate";
|
|
404
|
+
db: string;
|
|
405
|
+
} | {
|
|
406
|
+
stage: "blocking" | "async";
|
|
407
|
+
entryId?: string;
|
|
408
|
+
spanEnd?: string;
|
|
409
|
+
});
|
|
410
|
+
export type CompactionAwaitedRow = LcmRowBase & {
|
|
411
|
+
kind: "compaction-awaited";
|
|
412
|
+
elapsedMs: number;
|
|
413
|
+
timedOut: boolean;
|
|
414
|
+
};
|
|
415
|
+
export type CompactionQueuedRow = LcmRowBase & {
|
|
416
|
+
kind: "compaction-queued";
|
|
417
|
+
entryId: string;
|
|
418
|
+
queued: number;
|
|
419
|
+
};
|
|
420
|
+
export type CompactionQueuedDroppedRow = LcmRowBase & {
|
|
421
|
+
kind: "compaction-queued-dropped";
|
|
422
|
+
entryId: string;
|
|
423
|
+
reason: "store-reset";
|
|
424
|
+
};
|
|
425
|
+
export type CompactionTimeoutClearedRow = LcmRowBase & {
|
|
426
|
+
kind: "compaction-timeout-cleared";
|
|
427
|
+
elapsedMs: number;
|
|
428
|
+
};
|
|
429
|
+
/** A pass whose result was computed under conditions the session no longer
|
|
430
|
+
* holds, so the rows it wrote were deleted instead of applied. `changed` is
|
|
431
|
+
* empty when only the model moved. */
|
|
432
|
+
export type CompactionStaleRow = LcmRowBase & {
|
|
433
|
+
kind: "compaction-stale";
|
|
434
|
+
reason: "model-changed" | "settings-changed";
|
|
435
|
+
changed: string[];
|
|
436
|
+
stage: SummarizerStage;
|
|
437
|
+
entryId: string;
|
|
438
|
+
span: MetricSpan;
|
|
439
|
+
dropped: number;
|
|
440
|
+
};
|
|
441
|
+
/** Rows a pass wrote that never became memory: what a dead or stale handle left,
|
|
442
|
+
* reaped by the next run, or dropped at commit because a child they named was
|
|
443
|
+
* gone. The count is the work that has to be done again. */
|
|
444
|
+
export type CompactionAbandonedRow = LcmRowBase & {
|
|
445
|
+
kind: "compaction-abandoned";
|
|
446
|
+
stage: SummarizerStage;
|
|
447
|
+
dropped: number;
|
|
448
|
+
};
|
|
449
|
+
/** Why a pass produced no text. `no-entries` is a span the store holds no message
|
|
450
|
+
* row for, and `covered` is one whose every message is already memory or another
|
|
451
|
+
* live run's. Neither is a failure, so neither is a `compaction-error`. */
|
|
452
|
+
export type CompactionNoopReason = "no-entries" | "covered";
|
|
453
|
+
/** A pass that stored nothing, with the span it was given. The walk was paid for,
|
|
454
|
+
* so the count is worth keeping even when the outcome is correct. */
|
|
455
|
+
export type CompactionNoopRow = LcmRowBase & {
|
|
456
|
+
kind: "compaction-noop";
|
|
457
|
+
stage: SummarizerStage;
|
|
458
|
+
reason: CompactionNoopReason;
|
|
459
|
+
ingested: number;
|
|
460
|
+
span?: MetricSpan;
|
|
461
|
+
};
|
|
462
|
+
/** A store left behind by an older build, upgraded in place by `/lcm migrate` so
|
|
463
|
+
* cross-session search can read it. Only that command writes one: a search skips
|
|
464
|
+
* a store it cannot read. */
|
|
465
|
+
export type StoreMigratedRow = LcmRowBase & {
|
|
466
|
+
kind: "store-migrated";
|
|
467
|
+
db: string;
|
|
468
|
+
fromGeneration: number;
|
|
469
|
+
toGeneration: number;
|
|
470
|
+
};
|
|
471
|
+
export type GcDeletedRow = LcmRowBase & {
|
|
472
|
+
kind: "gc-deleted";
|
|
473
|
+
db: string;
|
|
474
|
+
reason: string;
|
|
475
|
+
sizeBytes: number;
|
|
476
|
+
exportedTo: string;
|
|
477
|
+
};
|
|
478
|
+
export type BranchRemovedRow = LcmRowBase & {
|
|
479
|
+
kind: "branch-removed";
|
|
480
|
+
removed: number;
|
|
481
|
+
restored: number;
|
|
482
|
+
};
|
|
483
|
+
export type FileExternalizedRow = LcmRowBase & {
|
|
484
|
+
kind: "file-externalized";
|
|
485
|
+
entryId: string;
|
|
486
|
+
fileId: string;
|
|
487
|
+
path: string;
|
|
488
|
+
fileKind: string;
|
|
489
|
+
bytes: number;
|
|
490
|
+
};
|
|
491
|
+
/** A large body with no path behind it, so nothing could be externalized. */
|
|
492
|
+
export type LargeInlineRow = LcmRowBase & {
|
|
493
|
+
kind: "large-inline";
|
|
494
|
+
entryId: string;
|
|
495
|
+
chars: number;
|
|
496
|
+
};
|
|
497
|
+
export type RecallRow = LcmRowBase & {
|
|
498
|
+
kind: "recall";
|
|
499
|
+
tool: "lcm_grep" | "lcm_describe";
|
|
500
|
+
outcome: "hit" | "miss" | "error";
|
|
501
|
+
hits?: number;
|
|
502
|
+
id?: number;
|
|
503
|
+
depth?: number;
|
|
504
|
+
tokens?: number;
|
|
505
|
+
sessionsScanned?: number;
|
|
506
|
+
sessionsSkipped?: number;
|
|
507
|
+
};
|
|
508
|
+
/** One procedural long-context evaluation run: the
|
|
509
|
+
* seed and size of the generated context, and whether every needle stayed
|
|
510
|
+
* reachable. `fromProjection` is what the surface a model sees already carried,
|
|
511
|
+
* `fromRecall` is what the memory had to recover, and `failed` is the count that
|
|
512
|
+
* has to stay zero. */
|
|
513
|
+
export type EvalRow = LcmRowBase & {
|
|
514
|
+
kind: "eval";
|
|
515
|
+
seed: number;
|
|
516
|
+
turns: number;
|
|
517
|
+
needles: number;
|
|
518
|
+
passes: number;
|
|
519
|
+
depth: number;
|
|
520
|
+
fidelity: number;
|
|
521
|
+
fromProjection: number;
|
|
522
|
+
fromRecall: number;
|
|
523
|
+
failed: number;
|
|
524
|
+
surfaceTokens: number;
|
|
525
|
+
};
|
|
526
|
+
/** One bounded-reader run, with the provider calls it made. `scope` is set when
|
|
527
|
+
* the caller named a source instead of searching for it, and the id it named
|
|
528
|
+
* rides along, so an addressed read can be counted apart from a search without
|
|
529
|
+
* reading the tool's arguments back. */
|
|
530
|
+
export type RetrievalRow = LcmRowBase & {
|
|
531
|
+
kind: "retrieval";
|
|
532
|
+
queryChars: number;
|
|
533
|
+
budgetTokens: number;
|
|
534
|
+
steps: number;
|
|
535
|
+
retrievedTokens: number;
|
|
536
|
+
stop: string;
|
|
537
|
+
reason?: string;
|
|
538
|
+
scope?: "summary" | "entry";
|
|
539
|
+
summaryId?: number;
|
|
540
|
+
entryId?: string;
|
|
541
|
+
} & Partial<UsageCounters>;
|
|
542
|
+
export type ImportRow = MetricBase & {
|
|
543
|
+
event: "lcm";
|
|
544
|
+
kind: "import";
|
|
545
|
+
db: string;
|
|
546
|
+
} & ({
|
|
547
|
+
refused: string;
|
|
548
|
+
} | {
|
|
549
|
+
messagesInserted: number;
|
|
550
|
+
messagesSkipped: number;
|
|
551
|
+
summariesInserted: number;
|
|
552
|
+
summariesSkipped: number;
|
|
553
|
+
});
|
|
554
|
+
export type LcmMetricRecord = ContextDecisionRow | ProjectionOverBudgetRow | SwapAppliedRow | SwapBlockedRow | PiCompactionRow | EstimateCalibratedRow | WindowCorrectedRow | WindowFloorRow | ConfigProblemRow | SummaryInputClippedRow | SummaryFallbackStoredRow | SummaryStoredRow | SummarySkippedCoveredRow | SummarySkippedExistingRow | SummaryOversizedRow | SummaryReplacedRow | SummaryFinalTruncatedRow | CondensationStalledRow | CondensationReducedRow | CompactionCompleteRow | SummarizerErrorRow | SummarizerCappedRow | SummarizerFallbackRow | SummarizerModelRow | SummarizerUsageRow | CompactionErrorRow | CompactionAwaitedRow | CompactionQueuedRow | CompactionQueuedDroppedRow | CompactionTimeoutClearedRow | CompactionStaleRow | CompactionAbandonedRow | CompactionNoopRow | GcDeletedRow | StoreMigratedRow | BranchRemovedRow | FileExternalizedRow | LargeInlineRow | RecallRow | RetrievalRow | EvalRow | ImportRow;
|
|
555
|
+
export type MetricRecord = UsageRow | LcmMetricRecord;
|
|
556
|
+
//#endregion
|
|
557
|
+
//#region src/files.d.ts
|
|
558
|
+
/** Characters of a body's description that stay in the preview. The message row
|
|
559
|
+
* keeps every character, so this bounds what the summarizer reads, not what is
|
|
560
|
+
* retained. */
|
|
561
|
+
export declare const FILE_PREVIEW_CHARS = 1200;
|
|
562
|
+
export declare const FILE_KINDS: readonly ["json", "jsonl", "csv", "tsv", "code", "text"];
|
|
563
|
+
export type FileKind = (typeof FILE_KINDS)[number];
|
|
564
|
+
export declare function isFileKind(value: unknown): value is FileKind;
|
|
565
|
+
export interface FileDescriptor {
|
|
566
|
+
fileId: string;
|
|
567
|
+
path: string;
|
|
568
|
+
/** Characters retained in the message, not bytes on disk. */
|
|
569
|
+
bytes: number;
|
|
570
|
+
sha256: string;
|
|
571
|
+
kind: FileKind;
|
|
572
|
+
preview: string;
|
|
573
|
+
}
|
|
574
|
+
export declare function fileHandle(d: FileDescriptor): string;
|
|
575
|
+
export declare function filePreviewText(d: FileDescriptor): string;
|
|
576
|
+
export declare function describeFile(path: string, text: string): FileDescriptor;
|
|
577
|
+
/** What the body is, decided by structure rather than by extension alone: a
|
|
578
|
+
* `.ts` file holding a JSON array is JSON, and a `.txt` file holding JSON is
|
|
579
|
+
* JSON too. The parse is tried before the extension for that reason. */
|
|
580
|
+
export declare function kindOf(path: string, text: string): FileKind;
|
|
581
|
+
export declare function structureOf(kind: FileKind, path: string, text: string): string;
|
|
582
|
+
export declare function previewOf(kind: FileKind, path: string, text: string): string;
|
|
583
|
+
//#endregion
|
|
584
|
+
//#region src/pattern-scan.d.ts
|
|
585
|
+
/** One row the pattern path scans. The text is the stored message, whole: the
|
|
586
|
+
* FTS index holds a prefix, and reading only that is the defect this path
|
|
587
|
+
* exists to fix. */
|
|
588
|
+
export interface ScannedRow {
|
|
589
|
+
rowid: number;
|
|
590
|
+
entryId: string;
|
|
591
|
+
role: string;
|
|
592
|
+
text: string;
|
|
593
|
+
removed: boolean;
|
|
594
|
+
}
|
|
595
|
+
/** A row that matched, with the first match's position and how many the row
|
|
596
|
+
* held. `partial` is set when the row was longer than the per-row cap, so a
|
|
597
|
+
* reader knows the scan stopped before the end of that row. */
|
|
598
|
+
export interface PatternHit {
|
|
599
|
+
row: ScannedRow;
|
|
600
|
+
matches: number;
|
|
601
|
+
at: number;
|
|
602
|
+
partial: boolean;
|
|
603
|
+
scannedChars: number;
|
|
604
|
+
}
|
|
605
|
+
export type PatternStop = "end" | "deadline" | "page";
|
|
606
|
+
export interface PatternScanOutcome {
|
|
607
|
+
hits: PatternHit[];
|
|
608
|
+
rows: number;
|
|
609
|
+
partialRows: number;
|
|
610
|
+
more: boolean;
|
|
611
|
+
stopped: PatternStop;
|
|
612
|
+
lastRowId?: number;
|
|
613
|
+
lastEntryId?: string;
|
|
614
|
+
}
|
|
615
|
+
export interface PatternScanOptions {
|
|
616
|
+
pattern: CompiledPattern;
|
|
617
|
+
offset: number;
|
|
618
|
+
limit: number;
|
|
619
|
+
/** Characters of one row the scan reads. Above the FTS prefix on purpose,
|
|
620
|
+
* below the size of an externalized body, and named on the hit when it bites. */
|
|
621
|
+
perRowChars: number;
|
|
622
|
+
deadlineAt: number;
|
|
623
|
+
now: () => number;
|
|
624
|
+
}
|
|
625
|
+
/** The characters a caller can hand a regular expression. A row longer than this
|
|
626
|
+
* is read up to the cap and the hit says so: a partial scan that looks complete
|
|
627
|
+
* is the same defect as an index that stops at 100,000 characters. */
|
|
628
|
+
export declare const PATTERN_ROW_CHARS = 200000;
|
|
629
|
+
/** How long one pattern call may scan. Long enough for a session's messages at
|
|
630
|
+
* 200,000 characters a row, short enough that a pathological pattern is a slow
|
|
631
|
+
* call rather than a hung session. */
|
|
632
|
+
export declare const PATTERN_DEADLINE_MS = 1000;
|
|
633
|
+
export declare const PATTERN_PAGE_ROWS = 200;
|
|
634
|
+
/** Matches counted in one row before the scan moves on. A row that mentions a
|
|
635
|
+
* path a thousand times is one hit line, and counting the thousandth costs the
|
|
636
|
+
* deadline that the next row needs. */
|
|
637
|
+
export declare const MAX_MATCHES_PER_ROW = 100;
|
|
638
|
+
export declare function scanRows(rows: Iterable<ScannedRow>, opts: PatternScanOptions): PatternScanOutcome;
|
|
639
|
+
/** A pattern this module compiled and validated, so a scan cannot receive a
|
|
640
|
+
* caller's own stateful `RegExp`. */
|
|
641
|
+
export type CompiledPattern = RegExp & {
|
|
642
|
+
readonly __brand: "CompiledPattern";
|
|
643
|
+
};
|
|
644
|
+
export type PatternCompileResult = {
|
|
645
|
+
ok: true;
|
|
646
|
+
pattern: CompiledPattern;
|
|
647
|
+
} | {
|
|
648
|
+
ok: false;
|
|
649
|
+
reason: string;
|
|
650
|
+
};
|
|
651
|
+
/** Compile a caller's pattern, with `g` for iteration and `i` unless the caller
|
|
652
|
+
* asked for case sensitivity, which is the reading the FTS path already has. A
|
|
653
|
+
* refusal is returned before any row is read. */
|
|
654
|
+
export declare function compilePattern(pattern: string, caseSensitive: boolean): PatternCompileResult;
|
|
655
|
+
//#endregion
|
|
656
|
+
//#region src/store.d.ts
|
|
657
|
+
export type LcmRole = "user" | "assistant" | "toolResult" | "custom";
|
|
658
|
+
/** Any role string Pi may emit (custom, branchSummary, bashExecution...) folds to the four stored roles. */
|
|
659
|
+
export declare function normalizeRole(r: string): LcmRole;
|
|
660
|
+
export interface StoredMessage {
|
|
661
|
+
id: number;
|
|
662
|
+
entryId: string;
|
|
663
|
+
role: LcmRole;
|
|
664
|
+
text: string;
|
|
665
|
+
tokens: number;
|
|
666
|
+
timestamp: number;
|
|
667
|
+
/** JSON of the content blocks, present only when `text` is not a faithful
|
|
668
|
+
* rendering of them (a thinking block, an image, a signature). Absent means
|
|
669
|
+
* the derived text is the whole message. */
|
|
670
|
+
payload?: string;
|
|
671
|
+
/** Set when the entry has left the active branch. The bytes stay; search
|
|
672
|
+
* excludes the row unless a caller asks for it. */
|
|
673
|
+
removedAt?: number;
|
|
674
|
+
}
|
|
675
|
+
export interface GrepHit {
|
|
676
|
+
entryId: string;
|
|
677
|
+
role: LcmRole;
|
|
678
|
+
text: string;
|
|
679
|
+
/** The entry has left the active branch; its text is retained. */
|
|
680
|
+
removed?: boolean;
|
|
681
|
+
coveringSummaryId?: number;
|
|
682
|
+
}
|
|
683
|
+
export interface SummaryNode {
|
|
684
|
+
id: number;
|
|
685
|
+
kind: "leaf" | "condensed";
|
|
686
|
+
text: string;
|
|
687
|
+
tokens: number;
|
|
688
|
+
depth: number;
|
|
689
|
+
firstEntryId: string;
|
|
690
|
+
lastEntryId: string;
|
|
691
|
+
createdAt: number;
|
|
692
|
+
/** A richer sibling text the ladder produced and did not store as primary,
|
|
693
|
+
* so the projection can serve more detail when the budget has room. */
|
|
694
|
+
thoroughText?: string;
|
|
695
|
+
thoroughTokens?: number;
|
|
696
|
+
}
|
|
697
|
+
export interface IntegritySnapshot {
|
|
698
|
+
ftsOk: boolean;
|
|
699
|
+
messageRows: number;
|
|
700
|
+
shapes: Array<{
|
|
701
|
+
id: number;
|
|
702
|
+
kind: "leaf" | "condensed";
|
|
703
|
+
depth: number;
|
|
704
|
+
messages: number;
|
|
705
|
+
children: number;
|
|
706
|
+
}>;
|
|
707
|
+
spans: Array<{
|
|
708
|
+
id: number;
|
|
709
|
+
first: string;
|
|
710
|
+
last: string;
|
|
711
|
+
firstRowid: number | null;
|
|
712
|
+
lastRowid: number | null;
|
|
713
|
+
}>;
|
|
714
|
+
duplicateSpans: Array<{
|
|
715
|
+
first: string;
|
|
716
|
+
last: string;
|
|
717
|
+
kind: string;
|
|
718
|
+
count: number;
|
|
719
|
+
}>;
|
|
720
|
+
orphanChildParents: number[];
|
|
721
|
+
deadRuns: Array<{
|
|
722
|
+
session: string | null;
|
|
723
|
+
rows: number[];
|
|
724
|
+
}>;
|
|
725
|
+
untrackedPending: number[];
|
|
726
|
+
foreignKeyViolations: number;
|
|
727
|
+
schemaVersion: number;
|
|
728
|
+
uncoveredMessages: number;
|
|
729
|
+
}
|
|
730
|
+
/** A fresh row, or the row that already owns the span; `existingTokens` comes
|
|
731
|
+
* from that row, so a caller reports the collision, not its intended write.
|
|
732
|
+
* `own` says whether the run that asked may count this row as its own and replace
|
|
733
|
+
* its text: a row it wrote, a committed row, or its own pending row. Another live
|
|
734
|
+
* run's pending row is none of those. */
|
|
735
|
+
export type InsertSummaryResult = {
|
|
736
|
+
id: number;
|
|
737
|
+
created: true;
|
|
738
|
+
own: true;
|
|
739
|
+
} | {
|
|
740
|
+
id: number;
|
|
741
|
+
created: false;
|
|
742
|
+
own: boolean;
|
|
743
|
+
existingTokens: number;
|
|
744
|
+
/** The colliding row's summary text. A caller whose view cannot read the row
|
|
745
|
+
* still has to answer for the span it was given, and this is that span's
|
|
746
|
+
* summary to reuse. */
|
|
747
|
+
existingText: string;
|
|
748
|
+
foreignLive: boolean;
|
|
749
|
+
};
|
|
750
|
+
/** What a caller hands `insertSummary`. `run` stamps the row pending for that run;
|
|
751
|
+
* without it the row is committed memory. */
|
|
752
|
+
export type InsertSummaryInput = {
|
|
753
|
+
kind: "leaf" | "condensed";
|
|
754
|
+
text: string;
|
|
755
|
+
tokens: number;
|
|
756
|
+
depth: number;
|
|
757
|
+
firstEntryId: string;
|
|
758
|
+
lastEntryId: string;
|
|
759
|
+
messageIds?: number[];
|
|
760
|
+
childSummaryIds?: number[];
|
|
761
|
+
fileIds?: string[];
|
|
762
|
+
thoroughText?: string;
|
|
763
|
+
run?: string;
|
|
764
|
+
};
|
|
765
|
+
/** What a compaction pass needs from the store: the committed memory plus, when
|
|
766
|
+
* the pass reads through `LcmStore.forRun`, the rows of its own run. `LcmStore`
|
|
767
|
+
* satisfies this with the committed-only meaning. */
|
|
768
|
+
export interface PassStore extends IngestStore {
|
|
769
|
+
/** The branch range a set of entry ids occupies: the oldest and the newest row
|
|
770
|
+
* the store holds among them, or null when it holds none. Pi's context array
|
|
771
|
+
* is not branch order (it hoists the newest compaction entry to the front),
|
|
772
|
+
* while a span is a row range, so the store, whose rows are appended in
|
|
773
|
+
* branch order, is what decides which end comes first. */
|
|
774
|
+
spanBounds(entryIds: readonly string[]): {
|
|
775
|
+
firstEntryId: string;
|
|
776
|
+
lastEntryId: string;
|
|
777
|
+
} | null;
|
|
778
|
+
uncoveredMessagesInSpan(firstEntryId: string, lastEntryId: string): StoredMessage[];
|
|
779
|
+
filesForMessages(ids: readonly number[]): Array<FileDescriptor & {
|
|
780
|
+
messageId: number;
|
|
781
|
+
}>;
|
|
782
|
+
frontier(firstEntryId: string, lastEntryId: string): SummaryNode[];
|
|
783
|
+
getSummary(id: number): SummaryNode | undefined;
|
|
784
|
+
parentSummaryIds(summaryId: number): number[];
|
|
785
|
+
fileIdsForSummaries(ids: readonly number[]): string[];
|
|
786
|
+
insertSummary(s: InsertSummaryInput): InsertSummaryResult;
|
|
787
|
+
updateSummaryText(id: number, text: string, tokens: number): void;
|
|
788
|
+
setThoroughText(id: number, thoroughText: string): void;
|
|
789
|
+
}
|
|
790
|
+
export interface IngestStore {
|
|
791
|
+
hasEntry(entryId: string): boolean;
|
|
792
|
+
insertMessage(m: {
|
|
793
|
+
entryId: string;
|
|
794
|
+
role: LcmRole;
|
|
795
|
+
text: string;
|
|
796
|
+
tokens: number;
|
|
797
|
+
timestamp: number;
|
|
798
|
+
payload?: string;
|
|
799
|
+
}): number;
|
|
800
|
+
insertFile(d: FileDescriptor, firstEntryId: string): void;
|
|
801
|
+
linkMessageFile(messageId: number, fileId: string): void;
|
|
802
|
+
}
|
|
803
|
+
/** Characters of a message the full-text index covers. Storage is verbatim, so
|
|
804
|
+
* this bounds the index and not the row. */
|
|
805
|
+
export declare const FTS_PREFIX_CHARS = 100000;
|
|
806
|
+
/** First published schema: pre-publish development stores were wiped, so every
|
|
807
|
+
* store in the wild starts here and later generations count from this one. */
|
|
808
|
+
export declare const SCHEMA_VERSION = 1;
|
|
809
|
+
/** A store left behind by an older build. A read-only open refuses it rather than
|
|
810
|
+
* migrating it: a search must not rewrite a file it is only reading, and a store
|
|
811
|
+
* that cannot answer this build's statements has to be a reported skip. */
|
|
812
|
+
export declare class StaleStoreError extends Error {
|
|
813
|
+
readonly generation: number;
|
|
814
|
+
constructor(generation: number);
|
|
815
|
+
}
|
|
816
|
+
/** What a store knows about a model rather than about a session: the token ratio
|
|
817
|
+
* measured for it against Pi's reported usage, and the context window a provider
|
|
818
|
+
* stated when it refused an over-window request. A null ratio is a measurement that
|
|
819
|
+
* disagree with itself, which is a state the estimator reads as "keep the
|
|
820
|
+
* constant" and the report names. A null window is a model no provider has
|
|
821
|
+
* corrected, which leaves Pi's belief in place. */
|
|
822
|
+
export interface ModelState {
|
|
823
|
+
charsPerToken: number | null;
|
|
824
|
+
samples: number;
|
|
825
|
+
contextWindow: number | null;
|
|
826
|
+
windowSource: string | null;
|
|
827
|
+
}
|
|
828
|
+
export declare const WINDOW_SOURCE_PROVIDER = "provider-error";
|
|
829
|
+
export declare class LcmStore {
|
|
830
|
+
private db;
|
|
831
|
+
private dbPath;
|
|
832
|
+
/** Opened for reading a past session's store: no migration ran, so `generation`
|
|
833
|
+
* is the file's own and a caller may have to skip it. */
|
|
834
|
+
private readOnly;
|
|
835
|
+
/** The schema generation this file carries, which read-only opens do not
|
|
836
|
+
* change. A writer holds `SCHEMA_VERSION` by the time its constructor returns. */
|
|
837
|
+
readonly generation: number;
|
|
838
|
+
/** True while a `transaction` callback is running, so a reentrant call joins
|
|
839
|
+
* the open transaction instead of nesting one, which SQLite rejects. */
|
|
840
|
+
private inTransaction;
|
|
841
|
+
/** Tokens of the runs this handle has open. Two runs can overlap inside one
|
|
842
|
+
* handle, because the blocking path stops waiting for the async pass, so a
|
|
843
|
+
* token names a run and not the handle. Whether a run this handle does not
|
|
844
|
+
* hold is still running is read from the `runs` table, never from here. */
|
|
845
|
+
private liveTokens;
|
|
846
|
+
private reapedInRun;
|
|
847
|
+
private stmts;
|
|
848
|
+
constructor(dbPath: string, opts?: {
|
|
849
|
+
readOnly?: boolean;
|
|
850
|
+
queryOnly?: boolean;
|
|
851
|
+
});
|
|
852
|
+
/** Close the handle: no run of this handle survives it, and the connection is
|
|
853
|
+
* released even when a run could not be abandoned, so a failure here cannot
|
|
854
|
+
* leave a connection and a set of invisible rows behind. */
|
|
855
|
+
close(): void;
|
|
856
|
+
/** Open a run: rows written with its token are pending until `commitRun`, and
|
|
857
|
+
* the row this writes is what lets another handle tell it from a run whose
|
|
858
|
+
* writer is gone. Reaps what such a run left, which this handle cannot be. */
|
|
859
|
+
openRun(opts?: {
|
|
860
|
+
session?: string;
|
|
861
|
+
}): {
|
|
862
|
+
token: string;
|
|
863
|
+
dropped: number;
|
|
864
|
+
};
|
|
865
|
+
/** True while any run is open whose writer is still there, this handle's or
|
|
866
|
+
* another process's, which is what an import has to wait for. */
|
|
867
|
+
hasLiveRun(): boolean;
|
|
868
|
+
/** Commit a run: drop its pending rows whose child no longer resolves, then flip
|
|
869
|
+
* the rest. One transaction, so a reader never sees a half-committed run, and
|
|
870
|
+
* idempotent for a token this handle no longer holds. */
|
|
871
|
+
commitRun(token: string): {
|
|
872
|
+
committed: number;
|
|
873
|
+
dropped: number;
|
|
874
|
+
};
|
|
875
|
+
/** Delete a run's pending rows and close it. Idempotent, so a `finally` can call
|
|
876
|
+
* it after a commit has already closed the run. */
|
|
877
|
+
abandonRun(token: string): number;
|
|
878
|
+
/** A view of this store for one run: the committed memory plus that run's own
|
|
879
|
+
* rows, which is what a pass must read to see its own writes. */
|
|
880
|
+
forRun(token: string): PassStore;
|
|
881
|
+
/** True while the run that wrote a row is still going somewhere, as far as this
|
|
882
|
+
* store can tell. A run this handle holds is live; a run written by this
|
|
883
|
+
* process is live, because a pid cannot separate two handles in one process and
|
|
884
|
+
* refusing to delete is the safe answer; a run written by another process is
|
|
885
|
+
* live while its pid is. A row whose run has no record at all was written
|
|
886
|
+
* without one, which no open path does, so nothing owns it. */
|
|
887
|
+
private isRunLive;
|
|
888
|
+
/** Delete what a run left behind whose writer is gone, and its record with it.
|
|
889
|
+
* A live owner's rows are never touched, whatever handle asks, which is what
|
|
890
|
+
* makes a second handle safe to open. No reader can see the rows this removes,
|
|
891
|
+
* and each one holds the span it claims. */
|
|
892
|
+
private reapPending;
|
|
893
|
+
/** Delete this run's pending rows whose child no longer exists, deepest first: a
|
|
894
|
+
* parent that loses a child must not leave a grandparent behind. A child can go
|
|
895
|
+
* away between the frontier read that selected it and this commit. */
|
|
896
|
+
private dropUnresolvable;
|
|
897
|
+
hasEntry(entryId: string): boolean;
|
|
898
|
+
insertMessage(m: {
|
|
899
|
+
entryId: string;
|
|
900
|
+
role: LcmRole;
|
|
901
|
+
text: string;
|
|
902
|
+
tokens: number;
|
|
903
|
+
timestamp: number;
|
|
904
|
+
payload?: string;
|
|
905
|
+
}): number;
|
|
906
|
+
/** One write transaction around a group of statements, so a compaction is
|
|
907
|
+
* atomic: the node row, its provenance rows and its file links either all
|
|
908
|
+
* land or none do. `node:sqlite` ships no transaction helper, so the
|
|
909
|
+
* statements are explicit. A reentrant call joins the open transaction: the
|
|
910
|
+
* caller that opened it owns the commit, so an inner throw rolls the whole
|
|
911
|
+
* group back rather than leaving a half-applied write. */
|
|
912
|
+
transaction<T>(fn: () => T): T;
|
|
913
|
+
insertSummary(s: InsertSummaryInput): InsertSummaryResult;
|
|
914
|
+
insertFile(d: FileDescriptor, firstEntryId: string): void;
|
|
915
|
+
linkMessageFile(messageId: number, fileId: string): void;
|
|
916
|
+
filesForMessages(ids: readonly number[]): Array<FileDescriptor & {
|
|
917
|
+
messageId: number;
|
|
918
|
+
}>;
|
|
919
|
+
fileIdsForMessages(ids: readonly number[]): string[];
|
|
920
|
+
fileIdsForSummaries(ids: readonly number[]): string[];
|
|
921
|
+
filesForSummaries(ids: readonly number[]): FileDescriptor[];
|
|
922
|
+
allFiles(): Array<FileDescriptor & {
|
|
923
|
+
entryIds: string[];
|
|
924
|
+
}>;
|
|
925
|
+
/** Replace a node's text in place: the span, provenance, and created_at stay. */
|
|
926
|
+
updateSummaryText(id: number, text: string, tokens: number): void;
|
|
927
|
+
setThoroughText(id: number, thoroughText: string): void;
|
|
928
|
+
messagesInSpan(firstEntryId: string, lastEntryId: string): StoredMessage[];
|
|
929
|
+
spanBounds(entryIds: readonly string[]): {
|
|
930
|
+
firstEntryId: string;
|
|
931
|
+
lastEntryId: string;
|
|
932
|
+
} | null;
|
|
933
|
+
uncoveredMessagesInSpan(firstEntryId: string, lastEntryId: string, run?: string): StoredMessage[];
|
|
934
|
+
/** Minimal top-level cover: no summary's child, oldest first, dropping any node
|
|
935
|
+
* whose span sits inside the previous kept one, which is what keeps the cover
|
|
936
|
+
* non-nested: condensation's group span must be one no row already holds. */
|
|
937
|
+
frontier(firstEntryId: string, lastEntryId: string, run?: string): SummaryNode[];
|
|
938
|
+
grep(pattern: string, opts?: {
|
|
939
|
+
limit?: number;
|
|
940
|
+
offset?: number;
|
|
941
|
+
withinMessageIds?: ReadonlySet<number>;
|
|
942
|
+
includeRemoved?: boolean;
|
|
943
|
+
}): GrepHit[];
|
|
944
|
+
grepCount(pattern: string, opts?: {
|
|
945
|
+
withinMessageIds?: ReadonlySet<number>;
|
|
946
|
+
includeRemoved?: boolean;
|
|
947
|
+
}): number | undefined;
|
|
948
|
+
messagePage(afterRowId: number, limit: number, opts?: {
|
|
949
|
+
withinMessageIds?: ReadonlySet<number>;
|
|
950
|
+
includeRemoved?: boolean;
|
|
951
|
+
}): ScannedRow[];
|
|
952
|
+
coveringSummaryId(entryId: string): number | undefined;
|
|
953
|
+
messageByEntryId(entryId: string): StoredMessage | undefined;
|
|
954
|
+
parentSummaryIds(summaryId: number, run?: string): number[];
|
|
955
|
+
childSummaryIds(summaryId: number): number[];
|
|
956
|
+
getSummary(id: number, run?: string): SummaryNode | undefined;
|
|
957
|
+
/** Bounded by a visited set, so a corrupt cycle cannot recurse forever. */
|
|
958
|
+
coveredMessageIds(summaryId: number, seen?: Set<number>): number[];
|
|
959
|
+
integritySnapshot(): IntegritySnapshot;
|
|
960
|
+
/** Restore the full-text index. The schema is re-applied first, because a
|
|
961
|
+
* dropped virtual table cannot be rebuilt, only recreated, and the DDL that
|
|
962
|
+
* knows how is the schema itself. Everything in it is `IF NOT EXISTS` apart
|
|
963
|
+
* from the two triggers, which are dropped and recreated on purpose. */
|
|
964
|
+
rebuildFts(): void;
|
|
965
|
+
deleteOrphanSpans(): number;
|
|
966
|
+
deleteOrphanChildren(): number;
|
|
967
|
+
reapDeadRuns(): number;
|
|
968
|
+
/** When one span is claimed twice, the row that carries provenance wins over
|
|
969
|
+
* one that cannot be expanded, and the newest wins between equals. */
|
|
970
|
+
dedupeSpans(): number;
|
|
971
|
+
/** Mark entries that left the active branch, or clear the mark when a
|
|
972
|
+
* navigation brings them back. Returns the rows the call actually changed, so
|
|
973
|
+
* a metric can tell a real change from a no-op reconciliation. */
|
|
974
|
+
setRemoved(entryIds: readonly string[], removedAt: number | null): number;
|
|
975
|
+
removedMessageCount(): number;
|
|
976
|
+
messagesByIds(ids: number[]): StoredMessage[];
|
|
977
|
+
allMessages(): StoredMessage[];
|
|
978
|
+
entrySpan(): {
|
|
979
|
+
first: string;
|
|
980
|
+
last: string;
|
|
981
|
+
} | undefined;
|
|
982
|
+
coveredSize(summaryId: number): {
|
|
983
|
+
tokens: number;
|
|
984
|
+
messages: number;
|
|
985
|
+
};
|
|
986
|
+
allSummaries(): SummaryNode[];
|
|
987
|
+
messageCount(): number;
|
|
988
|
+
backup(backupPath: string): void;
|
|
989
|
+
/** The calibration recorded for a model. `undefined` is a model nothing has
|
|
990
|
+
* sampled; a null ratio is one whose samples disagreed, which the caller reads
|
|
991
|
+
* as "keep the constant" rather than "no data". */
|
|
992
|
+
getModelState(modelKey: string): ModelState | undefined;
|
|
993
|
+
setModelState(modelKey: string, state: Pick<ModelState, "charsPerToken" | "samples">): void;
|
|
994
|
+
/** Records the window a provider stated. Its own statement rather than part of
|
|
995
|
+
* `setModelState`, because the two are learned at different moments: a later
|
|
996
|
+
* calibration must not clear a window, and a window must not touch samples. */
|
|
997
|
+
setModelWindow(modelKey: string, contextWindow: number, source: string): void;
|
|
998
|
+
stats(): {
|
|
999
|
+
messages: number;
|
|
1000
|
+
summaries: number;
|
|
1001
|
+
dbBytes: number;
|
|
1002
|
+
payloadMessages: number;
|
|
1003
|
+
payloadBytes: number;
|
|
1004
|
+
files: number;
|
|
1005
|
+
fileBytes: number;
|
|
1006
|
+
removedMessages: number;
|
|
1007
|
+
};
|
|
1008
|
+
}
|
|
1009
|
+
//#endregion
|
|
1010
|
+
//#region src/summarize.d.ts
|
|
1011
|
+
export interface LlmComplete {
|
|
1012
|
+
(systemPrompt: string, userText: string, maxTokens: number): Promise<string>;
|
|
1013
|
+
}
|
|
1014
|
+
export interface EscalationResult {
|
|
1015
|
+
text: string;
|
|
1016
|
+
level: 1 | 2 | 3;
|
|
1017
|
+
sourceTokens: number;
|
|
1018
|
+
retried: boolean;
|
|
1019
|
+
thoroughText?: string;
|
|
1020
|
+
}
|
|
1021
|
+
export declare function summarizeWithEscalation(items: Array<{
|
|
1022
|
+
role: string;
|
|
1023
|
+
text: string;
|
|
1024
|
+
}>, targetTokens: number, llm: LlmComplete, session?: string): Promise<EscalationResult>;
|
|
1025
|
+
/** Summarize with the terse rung only, for a source that is already a summary:
|
|
1026
|
+
* the thorough ladder would spend two provider calls re-summarizing text that
|
|
1027
|
+
* was made thorough once. Here the terse answer is the only ask, so a result
|
|
1028
|
+
* that does not reduce the source falls to the deterministic truncate. */
|
|
1029
|
+
export declare function summarizeTerse(items: Array<{
|
|
1030
|
+
role: string;
|
|
1031
|
+
text: string;
|
|
1032
|
+
}>, targetTokens: number, llm: LlmComplete, session?: string): Promise<EscalationResult>;
|
|
1033
|
+
export declare function deterministicTruncate(text: string, targetTokens: number): string;
|
|
1034
|
+
//#endregion
|
|
1035
|
+
//#region src/compaction-engine.d.ts
|
|
1036
|
+
/** Deepest condensed node the DAG may grow. With MAX_DAG_FANOUT children this
|
|
1037
|
+
* covers 8^8 leaf chunks, so it is a structural assert, not a tuning knob: a
|
|
1038
|
+
* pass that cannot meet its budget reports `condensation-stalled` rather than
|
|
1039
|
+
* stopping here silently. */
|
|
1040
|
+
export declare const MAX_DAG_DEPTH = 8;
|
|
1041
|
+
export declare const MAX_DAG_FANOUT = 8;
|
|
1042
|
+
export interface CompactionSpanEntry {
|
|
1043
|
+
entryId: string;
|
|
1044
|
+
role: LcmRole;
|
|
1045
|
+
text: string;
|
|
1046
|
+
payload?: string;
|
|
1047
|
+
}
|
|
1048
|
+
export interface CompactionInput {
|
|
1049
|
+
span: readonly CompactionSpanEntry[];
|
|
1050
|
+
previousSummary?: string;
|
|
1051
|
+
targetTokens?: number;
|
|
1052
|
+
/** Where the returned summary must reach back to; defaults to the span start. */
|
|
1053
|
+
frontierFrom?: string;
|
|
1054
|
+
}
|
|
1055
|
+
export type CompactionOutcome = {
|
|
1056
|
+
kind: "stored";
|
|
1057
|
+
summaryText: string;
|
|
1058
|
+
leafSummaryIds: number[];
|
|
1059
|
+
condensedSummaryIds: number[];
|
|
1060
|
+
ingested: number;
|
|
1061
|
+
} |
|
|
1062
|
+
/** No text, and why. A span another live run claimed, a span an existing node
|
|
1063
|
+
* already covers, and a span the store holds no row for are all states a pass
|
|
1064
|
+
* is allowed to find itself in, so the caller decides what to do with the state
|
|
1065
|
+
* instead of receiving a throw. A pass in this state stored nothing: every row
|
|
1066
|
+
* it writes sits inside the span it renders.
|
|
1067
|
+
*/
|
|
1068
|
+
{
|
|
1069
|
+
kind: "nothing";
|
|
1070
|
+
reason: CompactionNoopReason;
|
|
1071
|
+
ingested: number;
|
|
1072
|
+
};
|
|
1073
|
+
export declare function runCompaction(store: PassStore, input: CompactionInput, llm: LlmComplete, opts?: {
|
|
1074
|
+
leafChunkTokens?: number;
|
|
1075
|
+
maxChunks?: number;
|
|
1076
|
+
/** Masks stored text on its way to a model: the summarizer input, every
|
|
1077
|
+
* recall view, and the summary this pass stores. It never changes what is
|
|
1078
|
+
* written to the messages table. */
|
|
1079
|
+
redact?: (text: string) => string;
|
|
1080
|
+
/** Session tag for every metric this pass writes; without it a
|
|
1081
|
+
* fallback event cannot be attributed to a session. */
|
|
1082
|
+
session?: string;
|
|
1083
|
+
maxDepth?: number;
|
|
1084
|
+
stage?: "blocking" | "async" | "direct";
|
|
1085
|
+
}): Promise<CompactionOutcome>;
|
|
1086
|
+
/** The next group to condense: the oldest run of adjacent same-depth nodes, at
|
|
1087
|
+
* the shallowest depth with a run of two or more, capped at MAX_DAG_FANOUT.
|
|
1088
|
+
* Shallowest first keeps the tree balanced and the depth cap unreachable, since
|
|
1089
|
+
* a parent mixing depths grows the frontier instead of shrinking it. */
|
|
1090
|
+
export declare function condensationGroup(frontier: readonly SummaryNode[], opts?: {
|
|
1091
|
+
maxDepth?: number;
|
|
1092
|
+
}): SummaryNode[] | undefined;
|
|
1093
|
+
//#endregion
|
|
1094
|
+
//#region src/context-policy.d.ts
|
|
1095
|
+
export interface ContextPolicyInput {
|
|
1096
|
+
rawMessages: readonly AssemblyMsg[];
|
|
1097
|
+
getCtxEntries: () => readonly AlignedEntry[];
|
|
1098
|
+
occupancy: number;
|
|
1099
|
+
tokens: number;
|
|
1100
|
+
/** True when `tokens` is the estimator's count rather than Pi's, because the
|
|
1101
|
+
* reported figure described a context other than the one on hand. */
|
|
1102
|
+
tokensEstimated?: true;
|
|
1103
|
+
contextWindow: number;
|
|
1104
|
+
/** The window the ratio thresholds are measured against, when it differs from
|
|
1105
|
+
* Pi's belief: a correction from a provider's overflow error, or the one-shot
|
|
1106
|
+
* floor a refused request proved. The belief stays in `contextWindow` because a
|
|
1107
|
+
* correction is not a window change the pin should be dropped over. */
|
|
1108
|
+
thresholdWindow?: number;
|
|
1109
|
+
modelKey: string;
|
|
1110
|
+
zone?: string;
|
|
1111
|
+
thresholds: ThresholdConfig;
|
|
1112
|
+
keepRecentTokens: number;
|
|
1113
|
+
summaryTokens: number;
|
|
1114
|
+
clampNotified: boolean;
|
|
1115
|
+
commitState: CommitState | null;
|
|
1116
|
+
pinnedModelKey: string;
|
|
1117
|
+
pinnedWindow: number;
|
|
1118
|
+
}
|
|
1119
|
+
export type BailReason = "nothing-to-age" | "no-boundary" | "tail-exceeds-entries" | "aged-empty" | "aged-unstored" | "monotonic-refused" | "tail-shrunk" | "tail-mismatch" | "projection-over-budget";
|
|
1120
|
+
export type NonEmpty<T> = readonly [T, ...T[]];
|
|
1121
|
+
export interface ContextPolicyBase {
|
|
1122
|
+
commitState: CommitState | null;
|
|
1123
|
+
metrics: LcmMetricRecord[];
|
|
1124
|
+
pinnedModelKey: string;
|
|
1125
|
+
pinnedWindow: number;
|
|
1126
|
+
clampNotified: boolean;
|
|
1127
|
+
clampWarning?: string;
|
|
1128
|
+
}
|
|
1129
|
+
export type ContextPolicyResult = ContextPolicyBase & ({
|
|
1130
|
+
outcome: "quiet";
|
|
1131
|
+
} | {
|
|
1132
|
+
outcome: "reapply";
|
|
1133
|
+
messages: AssemblyMsg[];
|
|
1134
|
+
} | {
|
|
1135
|
+
outcome: "swap";
|
|
1136
|
+
messages: AssemblyMsg[];
|
|
1137
|
+
} | {
|
|
1138
|
+
outcome: "bail";
|
|
1139
|
+
reason: BailReason;
|
|
1140
|
+
messages?: AssemblyMsg[];
|
|
1141
|
+
} | {
|
|
1142
|
+
outcome: "kick";
|
|
1143
|
+
reason: "summary-missing";
|
|
1144
|
+
aged: NonEmpty<AlignedEntry>;
|
|
1145
|
+
/** The span this kick is about, in the order the store resolves one. The
|
|
1146
|
+
* adapter keys its latch on it, so the latch and the coverage check
|
|
1147
|
+
* cannot disagree about which boundary is missing. */
|
|
1148
|
+
span: {
|
|
1149
|
+
firstEntryId: string;
|
|
1150
|
+
lastEntryId: string;
|
|
1151
|
+
};
|
|
1152
|
+
messages?: AssemblyMsg[];
|
|
1153
|
+
});
|
|
1154
|
+
export interface ContextPolicyDeps {
|
|
1155
|
+
thresholds: (cfg: ThresholdConfig, window: number) => ResolvedThresholds;
|
|
1156
|
+
nextAction: (state: CommitState | null, occupancy: number, swap: number, recut: number) => AssemblyAction;
|
|
1157
|
+
cutIndexFor: (msgs: readonly AssemblyMsg[], keepRecentTokens: number) => number | null;
|
|
1158
|
+
cutMayReplace: (previous: CommitState | null, newCutCount: number) => boolean;
|
|
1159
|
+
frontier: (firstEntryId: string, lastEntryId: string) => SummaryNode[];
|
|
1160
|
+
/** The aged set's own ends as a span: branch order, and only entries the store
|
|
1161
|
+
* holds. Pi's context array is not branch order (it hoists the newest
|
|
1162
|
+
* compaction entry to the front), and a span whose ends are inverted resolves
|
|
1163
|
+
* to nothing, so this answers with the end pair a store query can use. Null
|
|
1164
|
+
* when the store holds none of the aged entries, which is a state rather than
|
|
1165
|
+
* a kick. */
|
|
1166
|
+
spanBounds: (aged: NonEmpty<AlignedEntry>) => {
|
|
1167
|
+
firstEntryId: string;
|
|
1168
|
+
lastEntryId: string;
|
|
1169
|
+
} | null;
|
|
1170
|
+
buildSynthetic: (summaries: readonly RenderableSummary[]) => CommitState["synthetic"];
|
|
1171
|
+
/** The mask node text passes through on its way to a model. The store keeps
|
|
1172
|
+
* the bytes, so every read seam takes this rather than storing a masked
|
|
1173
|
+
* copy; `makeRedact` is the provider a real caller passes. */
|
|
1174
|
+
redact: (text: string) => string;
|
|
1175
|
+
}
|
|
1176
|
+
export interface FailedBoundary {
|
|
1177
|
+
attempts: number;
|
|
1178
|
+
lastAt: number;
|
|
1179
|
+
}
|
|
1180
|
+
export declare const KICK_MAX_ATTEMPTS = 3;
|
|
1181
|
+
export declare const KICK_BACKOFF_MS: number;
|
|
1182
|
+
export declare function shouldKickBoundary(latch: FailedBoundary | undefined, now: number): boolean;
|
|
1183
|
+
export declare function applyContextPolicy(input: ContextPolicyInput, deps: ContextPolicyDeps): ContextPolicyResult;
|
|
1184
|
+
export interface CompactionBounds {
|
|
1185
|
+
startEntryId?: string;
|
|
1186
|
+
firstKeptEntryId: string;
|
|
1187
|
+
}
|
|
1188
|
+
/** Blocking-regime span (session_before_compact), computed the way Pi's
|
|
1189
|
+
* prepareCompaction does: branch entries from the previous compaction's first
|
|
1190
|
+
* kept entry up to but excluding the new one, minus compaction rows. Entry ids,
|
|
1191
|
+
* not message text, so bash and branch-summary rows align. An unknown
|
|
1192
|
+
* firstKeptEntryId yields an empty span, so the caller falls back to Pi. */
|
|
1193
|
+
export declare function resolveCompactionSpan(branchMsgs: readonly AlignedEntry[], bounds: CompactionBounds): AlignedEntry[];
|
|
1194
|
+
//#endregion
|
|
1195
|
+
//#region src/integrity.d.ts
|
|
1196
|
+
export type FindingKind = "fts-index" | "summary-span" | "summary-shape" | "orphan-child" | "duplicate-span" | "foreign-key" | "schema-version" | "dead-run" | "coverage";
|
|
1197
|
+
export interface Finding {
|
|
1198
|
+
kind: FindingKind;
|
|
1199
|
+
/** `violation` is an invariant the DAG depends on; `coverage` is a count. */
|
|
1200
|
+
severity: "violation" | "info";
|
|
1201
|
+
count: number;
|
|
1202
|
+
detail: string;
|
|
1203
|
+
sample: string[];
|
|
1204
|
+
}
|
|
1205
|
+
export interface IntegrityReport {
|
|
1206
|
+
/** How many checks ran, so "no findings" cannot be confused with "no checks". */
|
|
1207
|
+
checks: number;
|
|
1208
|
+
findings: Finding[];
|
|
1209
|
+
}
|
|
1210
|
+
export declare function checkIntegrity(store: LcmStore): IntegrityReport;
|
|
1211
|
+
/** Fix what can be fixed without guessing: the derived index, rows whose span no
|
|
1212
|
+
* longer exists, nodes whose child no longer resolves, rows a run whose writer is
|
|
1213
|
+
* gone left behind, and predated duplicate spans. An inverted span is left
|
|
1214
|
+
* alone, because which end is right needs the session file. */
|
|
1215
|
+
export declare function repairIntegrity(store: LcmStore): string[];
|
|
1216
|
+
//#endregion
|
|
1217
|
+
//#region src/host.d.ts
|
|
1218
|
+
/** What one call was billed, in numbers, so a pass can sum its spend.
|
|
1219
|
+
* `UsageLike` in `metrics.ts` is the tolerant reader of the same facts at the
|
|
1220
|
+
* metrics boundary, and a `Usage` satisfies it. */
|
|
1221
|
+
export interface Usage {
|
|
1222
|
+
input: number;
|
|
1223
|
+
output: number;
|
|
1224
|
+
cacheRead: number;
|
|
1225
|
+
cacheWrite: number;
|
|
1226
|
+
/** A subset of `cacheWrite`, reported by hosts that split retention. */
|
|
1227
|
+
cacheWrite1h?: number;
|
|
1228
|
+
/** A subset of `output`, reported by hosts that expose the breakdown. */
|
|
1229
|
+
reasoning?: number;
|
|
1230
|
+
totalTokens: number;
|
|
1231
|
+
cost: {
|
|
1232
|
+
input: number;
|
|
1233
|
+
output: number;
|
|
1234
|
+
cacheRead: number;
|
|
1235
|
+
cacheWrite: number;
|
|
1236
|
+
total: number;
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
/** How core tells the operator something. A host with no UI is `none` rather
|
|
1240
|
+
* than a dropped callback, so a caller has one branch and no optional to miss. */
|
|
1241
|
+
export type Notices = {
|
|
1242
|
+
readonly kind: "none";
|
|
1243
|
+
} | {
|
|
1244
|
+
readonly kind: "ui";
|
|
1245
|
+
notify(text: string, level: "info" | "warning"): void;
|
|
1246
|
+
};
|
|
1247
|
+
/** One model the host can call. `key` is the spelling config zones and metric
|
|
1248
|
+
* rows already use; `model` is the host's own handle, opaque here. */
|
|
1249
|
+
export interface ModelRef<Model> {
|
|
1250
|
+
readonly key: string;
|
|
1251
|
+
readonly reasoning: boolean;
|
|
1252
|
+
readonly model: Model;
|
|
1253
|
+
}
|
|
1254
|
+
export interface ModelRequest {
|
|
1255
|
+
readonly provider: string;
|
|
1256
|
+
readonly id: string;
|
|
1257
|
+
}
|
|
1258
|
+
export interface CompletionRequest {
|
|
1259
|
+
readonly prompt: string;
|
|
1260
|
+
readonly maxTokens: number;
|
|
1261
|
+
}
|
|
1262
|
+
/** What a call came back with. `stop` is the host's own word, kept verbatim for
|
|
1263
|
+
* metric rows; `capped` is the host saying the answer was cut short, which is
|
|
1264
|
+
* the one thing a caller must never store. */
|
|
1265
|
+
export type ModelAnswer = {
|
|
1266
|
+
outcome: "answer";
|
|
1267
|
+
text: string;
|
|
1268
|
+
stop: string;
|
|
1269
|
+
usage: Usage | undefined;
|
|
1270
|
+
} | {
|
|
1271
|
+
outcome: "failed";
|
|
1272
|
+
reason: string;
|
|
1273
|
+
capped: boolean;
|
|
1274
|
+
stop: string;
|
|
1275
|
+
usage: Usage | undefined;
|
|
1276
|
+
};
|
|
1277
|
+
export interface ModelHost<Model> {
|
|
1278
|
+
readonly notices: Notices;
|
|
1279
|
+
session(): ModelRef<Model> | undefined;
|
|
1280
|
+
find(requested: ModelRequest): ModelRef<Model> | undefined;
|
|
1281
|
+
complete(model: ModelRef<Model>, request: CompletionRequest): Promise<ModelAnswer>;
|
|
1282
|
+
}
|
|
1283
|
+
/** A tool the reader may call. The schema is the host's, passed through rather
|
|
1284
|
+
* than rebuilt. */
|
|
1285
|
+
export interface ReaderTool {
|
|
1286
|
+
name: string;
|
|
1287
|
+
description: string;
|
|
1288
|
+
parameters: TSchema;
|
|
1289
|
+
}
|
|
1290
|
+
export interface ReaderCall {
|
|
1291
|
+
readonly id: string;
|
|
1292
|
+
readonly name: string;
|
|
1293
|
+
readonly arguments: Record<string, unknown>;
|
|
1294
|
+
}
|
|
1295
|
+
export interface ReaderResult {
|
|
1296
|
+
readonly call: ReaderCall;
|
|
1297
|
+
readonly text: string;
|
|
1298
|
+
}
|
|
1299
|
+
/** What one reader step produced. `reads` asks the caller to run the calls;
|
|
1300
|
+
* `answered` means the reader is done, with whatever text it accumulated. */
|
|
1301
|
+
export type ReaderReply = {
|
|
1302
|
+
outcome: "reads";
|
|
1303
|
+
calls: readonly ReaderCall[];
|
|
1304
|
+
text: string;
|
|
1305
|
+
usage: Usage | undefined;
|
|
1306
|
+
} | {
|
|
1307
|
+
outcome: "answered";
|
|
1308
|
+
text: string;
|
|
1309
|
+
usage: Usage | undefined;
|
|
1310
|
+
} | {
|
|
1311
|
+
outcome: "failed";
|
|
1312
|
+
reason: string;
|
|
1313
|
+
usage: Usage | undefined;
|
|
1314
|
+
};
|
|
1315
|
+
export interface ReaderRequest {
|
|
1316
|
+
readonly systemPrompt: string;
|
|
1317
|
+
readonly brief: string;
|
|
1318
|
+
readonly tools: readonly ReaderTool[];
|
|
1319
|
+
}
|
|
1320
|
+
/** One reader conversation. The host owns the transcript: `next` appends the
|
|
1321
|
+
* results it is handed, appends its own reply, and calls the model. */
|
|
1322
|
+
export interface Reading {
|
|
1323
|
+
next(results: readonly ReaderResult[]): Promise<ReaderReply>;
|
|
1324
|
+
}
|
|
1325
|
+
export interface Reader {
|
|
1326
|
+
begin(request: ReaderRequest): Reading;
|
|
1327
|
+
}
|
|
1328
|
+
//#endregion
|
|
1329
|
+
//#region src/llm.d.ts
|
|
1330
|
+
export interface SummarizerConfig {
|
|
1331
|
+
/** One model id, or an ordered fallback chain. `auto` is the session model. */
|
|
1332
|
+
summarizer?: string | string[];
|
|
1333
|
+
/** Tokens one request may cost, estimate plus output cap, before it is refused
|
|
1334
|
+
* rather than paid for. A backstop, not a throttle. */
|
|
1335
|
+
summarizerTokensPerCall?: number;
|
|
1336
|
+
/** Tokens one pass may spend across all its calls, counted from what the
|
|
1337
|
+
* provider reported. Reaching it finishes the pass mechanically. */
|
|
1338
|
+
summarizerTokensPerPass?: number;
|
|
1339
|
+
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Thinking tokens are charged against the completion budget, so a reasoning
|
|
1342
|
+
* model can spend the whole cap before it emits any text. The headroom sits on
|
|
1343
|
+
* top of the caller's request, so the requested output size does not change.
|
|
1344
|
+
*/
|
|
1345
|
+
export declare const REASONING_HEADROOM = 4096;
|
|
1346
|
+
export interface ResolvedChain<Model> {
|
|
1347
|
+
/** The models to try, in order and deduped. Empty when nothing resolves. */
|
|
1348
|
+
models: ModelRef<Model>[];
|
|
1349
|
+
/** Configured entries the host could not resolve, in order. */
|
|
1350
|
+
unresolved: string[];
|
|
1351
|
+
requested: string[];
|
|
1352
|
+
}
|
|
1353
|
+
/** Expand the configured chain: `auto` (the session model) goes last unless the
|
|
1354
|
+
* chain names it, so one bad entry still leaves the session able to summarize. */
|
|
1355
|
+
export declare function resolveModels<Model>(host: ModelHost<Model>, config: SummarizerConfig): ResolvedChain<Model>;
|
|
1356
|
+
/** The one rendering of a resolved chain. The level travels with the warning so
|
|
1357
|
+
* no caller can pick a different notify level for the same chain. */
|
|
1358
|
+
export type ChainReport = {
|
|
1359
|
+
level: "info";
|
|
1360
|
+
state: string;
|
|
1361
|
+
} | {
|
|
1362
|
+
level: "warning";
|
|
1363
|
+
state: string;
|
|
1364
|
+
warning: string;
|
|
1365
|
+
};
|
|
1366
|
+
export declare function describeSummarizerChain<Model>(chain: ResolvedChain<Model>): ChainReport;
|
|
1367
|
+
/** The span every call from one completer summarizes. The failure cap is per
|
|
1368
|
+
* span, so a completer is built with the span it will be asked about. */
|
|
1369
|
+
export interface SummarizedSpan {
|
|
1370
|
+
firstEntryId: string;
|
|
1371
|
+
lastEntryId: string;
|
|
1372
|
+
}
|
|
1373
|
+
export interface MakeLlmOptions {
|
|
1374
|
+
session: string;
|
|
1375
|
+
stage?: SummarizerStage;
|
|
1376
|
+
span: SummarizedSpan;
|
|
1377
|
+
}
|
|
1378
|
+
/** Failed walks in a row before a pass stops calling the model and lets the
|
|
1379
|
+
* ladder's deterministic truncate finish the span. One is too eager, because a
|
|
1380
|
+
* single transport blip would turn a whole span mechanical; past three, another
|
|
1381
|
+
* call carries no new information about the provider and costs its latency. */
|
|
1382
|
+
export declare const SUMMARIZER_FAILURE_CAP = 3;
|
|
1383
|
+
/** What one request may cost before it is refused instead of paid for. It exists
|
|
1384
|
+
* for a pathological chunk rather than for a normal one. */
|
|
1385
|
+
export declare const SUMMARIZER_CALL_TOKENS = 60000;
|
|
1386
|
+
/** What one pass may spend across all of its calls. */
|
|
1387
|
+
export declare const SUMMARIZER_PASS_TOKENS = 250000;
|
|
1388
|
+
export declare function retriableCategory(category: SummarizerFailureCategory): boolean;
|
|
1389
|
+
/** How long a span whose calls were capped is left alone. Shorter than the kick
|
|
1390
|
+
* backoff (`KICK_BACKOFF_MS`) on purpose: this guards a retry inside one turn,
|
|
1391
|
+
* it is not the breaker that stops re-kicks across turns. */
|
|
1392
|
+
export declare const FAILED_SPAN_TTL_MS = 60000;
|
|
1393
|
+
/** What a failed span is identified by: the two ends a caller asked about and
|
|
1394
|
+
* the chain it asked. A different model is a different attempt, not the same
|
|
1395
|
+
* failure, so the chain is part of the key. */
|
|
1396
|
+
export declare function spanFingerprint(span: SummarizedSpan, chain: readonly string[]): string;
|
|
1397
|
+
/**
|
|
1398
|
+
* A call that reached no usable answer. `category` is what the provider's own
|
|
1399
|
+
* text says: a best-effort read that answers `unknown` rather than guessing, and
|
|
1400
|
+
* `capped` when the caller should stop asking rather than the provider failing.
|
|
1401
|
+
*/
|
|
1402
|
+
export declare class SummarizerFailure extends Error {
|
|
1403
|
+
readonly category: SummarizerFailureCategory;
|
|
1404
|
+
constructor(message: string, category: SummarizerFailureCategory);
|
|
1405
|
+
}
|
|
1406
|
+
/** Which bucket a provider's error text falls in. A 5xx or a dropped connection
|
|
1407
|
+
* is `unavailable` before it is `timeout`, because the gateway's own timeout is
|
|
1408
|
+
* a request that never reached a model. */
|
|
1409
|
+
export declare function classifyFailure(text: string | undefined): SummarizerFailureCategory;
|
|
1410
|
+
export declare function makeLlm<Model>(host: ModelHost<Model>, config: SummarizerConfig, opts: MakeLlmOptions): ((systemPrompt: string, userText: string, maxTokens: number) => Promise<string>) | null;
|
|
1411
|
+
/** Sessions share the process, so each one gets to see the config it runs, and
|
|
1412
|
+
* a session that starts fresh should not inherit a refusal from the last one. */
|
|
1413
|
+
export declare function resetSummarizerState(): void;
|
|
1414
|
+
//#endregion
|
|
1415
|
+
//#region src/report.d.ts
|
|
1416
|
+
type Row = MetricRecord;
|
|
1417
|
+
type KeysOf<T> = T extends unknown ? keyof T : never;
|
|
1418
|
+
type ValueOf<T, K> = T extends unknown ? (K extends keyof T ? T[K] : never) : never;
|
|
1419
|
+
type AnyField = KeysOf<Row>;
|
|
1420
|
+
type FieldOf<K extends AnyField> = ValueOf<Row, K>;
|
|
1421
|
+
type RowFields = { [K in AnyField]?: FieldOf<K>; };
|
|
1422
|
+
export type MetricsEvent = RowFields & {
|
|
1423
|
+
ts?: number;
|
|
1424
|
+
[key: string]: unknown;
|
|
1425
|
+
};
|
|
1426
|
+
export declare function collectMetricsEvents(): MetricsEvent[];
|
|
1427
|
+
export interface DepthStat {
|
|
1428
|
+
depth: number;
|
|
1429
|
+
count: number;
|
|
1430
|
+
totalTokens: number;
|
|
1431
|
+
avgTokens: number;
|
|
1432
|
+
}
|
|
1433
|
+
export declare function depthStats(summaries: readonly {
|
|
1434
|
+
depth: number;
|
|
1435
|
+
tokens: number;
|
|
1436
|
+
}[]): DepthStat[];
|
|
1437
|
+
export declare function buildReport(events: readonly MetricsEvent[]): string;
|
|
1438
|
+
//#endregion
|
|
1439
|
+
//#region src/diagnose.d.ts
|
|
1440
|
+
export interface FactConfig {
|
|
1441
|
+
swapAtTokens?: number;
|
|
1442
|
+
recutAtTokens?: number;
|
|
1443
|
+
smartZone?: number;
|
|
1444
|
+
swapAtRatio?: number;
|
|
1445
|
+
recutAtRatio?: number;
|
|
1446
|
+
assemblyEnabled?: boolean;
|
|
1447
|
+
}
|
|
1448
|
+
export interface SessionFacts {
|
|
1449
|
+
stats: ReturnType<LcmStore["stats"]>;
|
|
1450
|
+
integrity: {
|
|
1451
|
+
checks: number;
|
|
1452
|
+
findings: Finding[];
|
|
1453
|
+
repairs: string[];
|
|
1454
|
+
};
|
|
1455
|
+
depthRows: DepthStat[];
|
|
1456
|
+
pin: CommitState | null;
|
|
1457
|
+
model: string;
|
|
1458
|
+
window: {
|
|
1459
|
+
tokens: number;
|
|
1460
|
+
effective: ResolvedThresholds;
|
|
1461
|
+
} | null;
|
|
1462
|
+
chain: ChainReport;
|
|
1463
|
+
zoneConfigured: boolean;
|
|
1464
|
+
config: FactConfig;
|
|
1465
|
+
keepRecentTokens: number;
|
|
1466
|
+
configPath?: string;
|
|
1467
|
+
}
|
|
1468
|
+
/**
|
|
1469
|
+
* The store, projection and model facts, in the order both readers print them.
|
|
1470
|
+
* Generated from numbers and config values, so nothing here needs a mask: no
|
|
1471
|
+
* line can carry a character a session produced.
|
|
1472
|
+
*/
|
|
1473
|
+
export declare function sessionFactLines(f: SessionFacts): string[];
|
|
1474
|
+
export declare const MAX_METRIC_ROWS = 25;
|
|
1475
|
+
export declare const MAX_BUNDLE_BYTES = 8192;
|
|
1476
|
+
export interface DiagnoseBundleInput {
|
|
1477
|
+
/** Resolved at print time, so a package that moved still names its version.
|
|
1478
|
+
* Each is a label the host composes, `name version`, because this core names
|
|
1479
|
+
* no product: the header a second harness prints names that harness. */
|
|
1480
|
+
versions: {
|
|
1481
|
+
plugin: () => string;
|
|
1482
|
+
host: () => string;
|
|
1483
|
+
};
|
|
1484
|
+
runtime: {
|
|
1485
|
+
node: string;
|
|
1486
|
+
platform: string;
|
|
1487
|
+
arch: string;
|
|
1488
|
+
mode: string;
|
|
1489
|
+
};
|
|
1490
|
+
/** Basenames and a version, never a path. */
|
|
1491
|
+
session: {
|
|
1492
|
+
store: string;
|
|
1493
|
+
file: string;
|
|
1494
|
+
format: string;
|
|
1495
|
+
};
|
|
1496
|
+
facts: SessionFacts;
|
|
1497
|
+
config: {
|
|
1498
|
+
effective: string;
|
|
1499
|
+
path: string;
|
|
1500
|
+
};
|
|
1501
|
+
/** Every metrics row for this session, oldest first. The bundle takes the
|
|
1502
|
+
* tail, so it does not have to guess how many there are. */
|
|
1503
|
+
metrics: readonly Record<string, unknown>[];
|
|
1504
|
+
redact: (text: string) => string;
|
|
1505
|
+
home: string;
|
|
1506
|
+
}
|
|
1507
|
+
/** The block a reporter pastes. Generated lines and foreign strings are kept
|
|
1508
|
+
* apart: a line this module builds is never masked, because masking is what
|
|
1509
|
+
* turns `modelKey` into `[REDACTED]`, and a value that came from outside is
|
|
1510
|
+
* always masked. Absolute paths get one of two treatments: `~` inside the home
|
|
1511
|
+
* directory, `<path>` outside it. */
|
|
1512
|
+
export declare function buildDiagnoseBundle(input: DiagnoseBundleInput): string;
|
|
1513
|
+
export declare function sanitizeRow(row: Record<string, unknown>, redact: (text: string) => string, home: string): Record<string, unknown>;
|
|
1514
|
+
/** The session file's format version, from the header line Pi rewrites on every
|
|
1515
|
+
* flush. A file that is missing, unreadable or not a session header renders the
|
|
1516
|
+
* honest fallback rather than a wrong number. */
|
|
1517
|
+
export declare function readSessionFormat(sessionFile: string | undefined): string;
|
|
1518
|
+
/** The host package's version, resolved from the caller's own module
|
|
1519
|
+
* (`import.meta.url`); a bundled engine cannot know where its host lives. */
|
|
1520
|
+
export declare function hostVersion(hostPackage: string, from: string | URL): string;
|
|
1521
|
+
/** The `version` of the nearest `package.json` named `name`, walking up from the
|
|
1522
|
+
* directory holding `from`. A manifest that is missing, unreadable, unparsable
|
|
1523
|
+
* or named something else is not the answer, so a wrong anchor fails the name
|
|
1524
|
+
* check instead of reporting a stranger's version. */
|
|
1525
|
+
export declare function packageVersionNear(from: string, name: string): string | undefined;
|
|
1526
|
+
//#endregion
|
|
1527
|
+
//#region src/estimate-tokens.d.ts
|
|
1528
|
+
export declare const CHARS_PER_TOKEN = 4;
|
|
1529
|
+
/** Samples below this count say nothing: with two points any line fits, and the
|
|
1530
|
+
* first turns of a session are exactly where the overhead has not cancelled. */
|
|
1531
|
+
export declare const CALIBRATION_MIN_SAMPLES = 3;
|
|
1532
|
+
/** Samples kept in memory. A cap keeps the median describing the recent turns of
|
|
1533
|
+
* the session rather than its whole history, which is what a model switch inside
|
|
1534
|
+
* one session would otherwise be unable to move. */
|
|
1535
|
+
export declare const CALIBRATION_SAMPLE_CAP = 8;
|
|
1536
|
+
/** One observation of the same session: how many characters this build counted in
|
|
1537
|
+
* the messages, and how many tokens Pi reported for the request that carried
|
|
1538
|
+
* them. Sampling differences rather than totals is what cancels the system prompt
|
|
1539
|
+
* and tool definitions, which this build never counts. */
|
|
1540
|
+
export interface EstimateSample {
|
|
1541
|
+
chars: number;
|
|
1542
|
+
tokens: number;
|
|
1543
|
+
}
|
|
1544
|
+
export declare function estimateTokens(text: string): number;
|
|
1545
|
+
export declare function tokensFromChars(chars: number): number;
|
|
1546
|
+
/** A reported count below this share of the estimate describes a context other
|
|
1547
|
+
* than the one on hand: Pi reports its last measurement, and after a resume that
|
|
1548
|
+
* measurement belongs to the previous process. The estimator's own error is
|
|
1549
|
+
* bounded by the calibration band, which is a factor of two in either direction
|
|
1550
|
+
* around the constant, so anything below a quarter is a different turn rather
|
|
1551
|
+
* than a miscount. A reported count too high only compacts early, which is why
|
|
1552
|
+
* this looks one way. */
|
|
1553
|
+
export declare const STALE_REPORT_SHARE = 0.25;
|
|
1554
|
+
export declare function calibrate(samples: readonly EstimateSample[]): number | null;
|
|
1555
|
+
/** A configured token budget in the units `estimateTokens` counts. The estimator
|
|
1556
|
+
* counts chars / 4, so a model whose real ratio is 6 needs 1.5 times the
|
|
1557
|
+
* configured number of estimated tokens accumulated before it has kept the
|
|
1558
|
+
* context the user asked for. Without a ratio the budget is left alone, which is
|
|
1559
|
+
* the constant's own reading of it. */
|
|
1560
|
+
export declare function scaledBudget(tokens: number, charsPerToken: number | null): number;
|
|
1561
|
+
//#endregion
|
|
1562
|
+
//#region src/eval.d.ts
|
|
1563
|
+
/** The seed CI runs. Fixed, so a change in the answer is a change in the engine
|
|
1564
|
+
* rather than a change in the fixture. */
|
|
1565
|
+
export declare const SMALL_SEED = 20260915;
|
|
1566
|
+
export interface EvalOptions {
|
|
1567
|
+
seed: number;
|
|
1568
|
+
turns: number;
|
|
1569
|
+
needles: number;
|
|
1570
|
+
fillerChars: number;
|
|
1571
|
+
}
|
|
1572
|
+
export interface EvalNeedle {
|
|
1573
|
+
id: string;
|
|
1574
|
+
entryId: string;
|
|
1575
|
+
value: number;
|
|
1576
|
+
/** The exact text stored in the entry, `N07=4821`. */
|
|
1577
|
+
line: string;
|
|
1578
|
+
}
|
|
1579
|
+
export interface EvalFixture {
|
|
1580
|
+
seed: number;
|
|
1581
|
+
entries: Array<{
|
|
1582
|
+
entryId: string;
|
|
1583
|
+
role: LcmRole;
|
|
1584
|
+
text: string;
|
|
1585
|
+
timestamp: number;
|
|
1586
|
+
}>;
|
|
1587
|
+
needles: EvalNeedle[];
|
|
1588
|
+
key: {
|
|
1589
|
+
needles: number;
|
|
1590
|
+
sum: number;
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
/** mulberry32: 32 bits of state, integer output, no dependency. The generator is
|
|
1594
|
+
* the fixture, so it has to be as frozen as the seed. */
|
|
1595
|
+
export declare function seededRandom(seed: number): () => number;
|
|
1596
|
+
/** Context of any length, with the answer key beside it. Needles are spread
|
|
1597
|
+
* across the turns rather than clustered at one end, so a pass that walks the
|
|
1598
|
+
* oldest span first meets needles at every depth. */
|
|
1599
|
+
export declare function generateContext(opts: EvalOptions): EvalFixture;
|
|
1600
|
+
/** How many uncovered messages one pass takes. Wide enough that a pass writes
|
|
1601
|
+
* several leaf chunks and then condenses them, so "N compactions" exercises the
|
|
1602
|
+
* DAG's second level and not only its first. */
|
|
1603
|
+
export declare const EVAL_SPAN_MESSAGES = 24;
|
|
1604
|
+
export declare const EVAL_LEAF_CHUNK_TOKENS = 2000;
|
|
1605
|
+
/** The token budget the pass must fit its frontier into, small on purpose: it is
|
|
1606
|
+
* what makes a pass condense the leaves it just wrote. */
|
|
1607
|
+
export declare const EVAL_TARGET_TOKENS = 400;
|
|
1608
|
+
export interface EvalResult {
|
|
1609
|
+
seed: number;
|
|
1610
|
+
turns: number;
|
|
1611
|
+
needles: number;
|
|
1612
|
+
passes: number;
|
|
1613
|
+
depth: number;
|
|
1614
|
+
summaries: number;
|
|
1615
|
+
condensed: number;
|
|
1616
|
+
fidelity: number;
|
|
1617
|
+
covered: number;
|
|
1618
|
+
readable: number;
|
|
1619
|
+
fromProjection: number;
|
|
1620
|
+
fromRecall: number;
|
|
1621
|
+
failed: number;
|
|
1622
|
+
surfaceTokens: number;
|
|
1623
|
+
answered: boolean;
|
|
1624
|
+
}
|
|
1625
|
+
export interface EvalRunOptions {
|
|
1626
|
+
passes: number;
|
|
1627
|
+
dbPath?: string;
|
|
1628
|
+
record?: boolean;
|
|
1629
|
+
}
|
|
1630
|
+
export declare function runEval(opts: EvalOptions, run: EvalRunOptions): Promise<EvalResult>;
|
|
1631
|
+
export declare function formatEvalTable(results: readonly EvalResult[]): string;
|
|
1632
|
+
//#endregion
|
|
1633
|
+
//#region src/export.d.ts
|
|
1634
|
+
/** The format this build writes, and the only one it reads. A range reads as a
|
|
1635
|
+
* compatibility promise while the row readers ignore the number, and the store,
|
|
1636
|
+
* not the file, holds the truth: a file that does not fit is re-exported. */
|
|
1637
|
+
export declare const EXPORT_VERSION = 1;
|
|
1638
|
+
/** A message row: the store's numeric id is not portable, so the entry id is
|
|
1639
|
+
* the handle an importer resolves through. */
|
|
1640
|
+
export interface ExportMessageRow {
|
|
1641
|
+
v: number;
|
|
1642
|
+
rowType: "message";
|
|
1643
|
+
entryId: string;
|
|
1644
|
+
role: LcmRole;
|
|
1645
|
+
text: string;
|
|
1646
|
+
tokens: number;
|
|
1647
|
+
timestamp: number;
|
|
1648
|
+
payload?: string;
|
|
1649
|
+
}
|
|
1650
|
+
/** A summary row. Provenance makes a summary usable: a leaf names the messages
|
|
1651
|
+
* it covers, a condensed node names children by file position, since ids are
|
|
1652
|
+
* assigned on import. `createdAt` is not carried, because the store assigns its
|
|
1653
|
+
* own and a field no import can restore would be a lie in the file. A run in
|
|
1654
|
+
* flight is not exported: what this returns is committed memory, so a pass still
|
|
1655
|
+
* holding pending rows is invisible here and an import waits for it. */
|
|
1656
|
+
export interface ExportSummaryRow {
|
|
1657
|
+
v: number;
|
|
1658
|
+
rowType: "summary";
|
|
1659
|
+
ordinal: number;
|
|
1660
|
+
kind: "leaf" | "condensed";
|
|
1661
|
+
text: string;
|
|
1662
|
+
tokens: number;
|
|
1663
|
+
depth: number;
|
|
1664
|
+
firstEntryId: string;
|
|
1665
|
+
lastEntryId: string;
|
|
1666
|
+
messageEntryIds: string[];
|
|
1667
|
+
childOrdinals: number[];
|
|
1668
|
+
}
|
|
1669
|
+
export type ExportRow = ExportMessageRow | ExportSummaryRow | ExportFileRow;
|
|
1670
|
+
export interface ExportFileRow {
|
|
1671
|
+
v: number;
|
|
1672
|
+
rowType: "file";
|
|
1673
|
+
fileId: string;
|
|
1674
|
+
path: string;
|
|
1675
|
+
bytes: number;
|
|
1676
|
+
sha256: string;
|
|
1677
|
+
kind: FileKind;
|
|
1678
|
+
preview: string;
|
|
1679
|
+
entryIds: string[];
|
|
1680
|
+
}
|
|
1681
|
+
export declare function exportTranscript(store: LcmStore): {
|
|
1682
|
+
messages: StoredMessage[];
|
|
1683
|
+
summaries: SummaryNode[];
|
|
1684
|
+
toJSONL: () => string;
|
|
1685
|
+
};
|
|
1686
|
+
//#endregion
|
|
1687
|
+
//#region src/format.d.ts
|
|
1688
|
+
export declare function fmtTokens(n: number): string;
|
|
1689
|
+
export declare function fmtBytes(n: number): string;
|
|
1690
|
+
//#endregion
|
|
1691
|
+
//#region src/home-paths.d.ts
|
|
1692
|
+
/** Escape hatch for a live session that carries a test signal for another
|
|
1693
|
+
* reason, so the guard can never brick one. */
|
|
1694
|
+
export declare const ALLOW_REAL_HOME_FLAG = "LCM_ALLOW_REAL_HOME";
|
|
1695
|
+
export declare function isUnderTempDir(path: string): boolean;
|
|
1696
|
+
/** Every path this extension touches under the user's home resolves here, so one
|
|
1697
|
+
* guard covers them all instead of one per call site. A test run refuses a path
|
|
1698
|
+
* outside its temp home rather than touching the real config. */
|
|
1699
|
+
export declare function lcmHomePath(...segments: string[]): string;
|
|
1700
|
+
//#endregion
|
|
1701
|
+
//#region src/import.d.ts
|
|
1702
|
+
export interface ImportResult {
|
|
1703
|
+
messagesInserted: number;
|
|
1704
|
+
messagesSkipped: number;
|
|
1705
|
+
summariesInserted: number;
|
|
1706
|
+
summariesSkipped: number;
|
|
1707
|
+
/** Set when the import was refused, with the text naming the reason. The
|
|
1708
|
+
* counters say what landed before it was: a refusal ahead of the summary loop
|
|
1709
|
+
* writes nothing, and one inside it names the rows that did land. */
|
|
1710
|
+
refused?: string;
|
|
1711
|
+
}
|
|
1712
|
+
export type ParsedExport = {
|
|
1713
|
+
rows: ExportRow[];
|
|
1714
|
+
} | {
|
|
1715
|
+
error: string;
|
|
1716
|
+
};
|
|
1717
|
+
/** Read a file written by `/lcm export`: every line is validated before any is
|
|
1718
|
+
* used, so a truncated or foreign file is refused without a partial import and
|
|
1719
|
+
* the caller can report a line number. */
|
|
1720
|
+
export declare function parseExport(text: string): ParsedExport;
|
|
1721
|
+
export declare function importTranscript(store: LcmStore, rows: readonly ExportRow[]): ImportResult;
|
|
1722
|
+
//#endregion
|
|
1723
|
+
//#region src/ingest.d.ts
|
|
1724
|
+
export interface IngestibleEntry {
|
|
1725
|
+
entryId: string;
|
|
1726
|
+
role: LcmRole;
|
|
1727
|
+
text: string;
|
|
1728
|
+
timestamp: number;
|
|
1729
|
+
payload?: string;
|
|
1730
|
+
/** The path a tool call named for this body, when one did. The only
|
|
1731
|
+
* deterministic signal that a body is a file, so the only externalization
|
|
1732
|
+
* trigger. */
|
|
1733
|
+
fileHint?: {
|
|
1734
|
+
path: string;
|
|
1735
|
+
};
|
|
1736
|
+
/** Pi's own head compaction, landing as a `custom` role because every role Pi
|
|
1737
|
+
* does not name folds to `custom`: the one observable event that the window
|
|
1738
|
+
* filled and LCM did not hold it. */
|
|
1739
|
+
piCompaction?: boolean;
|
|
1740
|
+
}
|
|
1741
|
+
export interface ExternalizedFile {
|
|
1742
|
+
entryId: string;
|
|
1743
|
+
fileId: string;
|
|
1744
|
+
path: string;
|
|
1745
|
+
kind: string;
|
|
1746
|
+
bytes: number;
|
|
1747
|
+
}
|
|
1748
|
+
export interface IngestStats {
|
|
1749
|
+
inserted: number;
|
|
1750
|
+
skipped: number;
|
|
1751
|
+
/** Entry ids inserted with `piCompaction`, so a re-scan cannot count one twice. */
|
|
1752
|
+
piCompactions: string[];
|
|
1753
|
+
files: ExternalizedFile[];
|
|
1754
|
+
largeInline: Array<{
|
|
1755
|
+
entryId: string;
|
|
1756
|
+
chars: number;
|
|
1757
|
+
}>;
|
|
1758
|
+
}
|
|
1759
|
+
export interface IngestWatermark {
|
|
1760
|
+
id: string;
|
|
1761
|
+
}
|
|
1762
|
+
export interface RawEntryLike {
|
|
1763
|
+
id?: string;
|
|
1764
|
+
}
|
|
1765
|
+
export declare function isIngestible(e: {
|
|
1766
|
+
entryId: string;
|
|
1767
|
+
text: string;
|
|
1768
|
+
}): boolean;
|
|
1769
|
+
/** The watermark a hook carries after considering `delta`: its last entry,
|
|
1770
|
+
* whether or not any of it was stored. A turn that carries nothing addressable
|
|
1771
|
+
* still moves the watermark, or the next turn rescans the whole session. */
|
|
1772
|
+
export declare function nextIngestWatermark(delta: Array<{
|
|
1773
|
+
entryId: string;
|
|
1774
|
+
}>): IngestWatermark | null;
|
|
1775
|
+
export interface RawIngestDelta {
|
|
1776
|
+
startIndex: number;
|
|
1777
|
+
reset: boolean;
|
|
1778
|
+
}
|
|
1779
|
+
export declare function computeRawIngestDelta(entries: readonly RawEntryLike[], watermark: IngestWatermark | null): RawIngestDelta;
|
|
1780
|
+
export declare function entryToText(content: unknown): string;
|
|
1781
|
+
/** The raw content blocks as JSON when `entryToText(content)` is not a faithful
|
|
1782
|
+
* rendering of them, `undefined` when the derived text is the whole message.
|
|
1783
|
+
* The whole array is stored, so a payload is never itself a lossy projection;
|
|
1784
|
+
* the rule about which messages get one is what keeps a text-only message at a
|
|
1785
|
+
* single copy. */
|
|
1786
|
+
export declare function payloadOf(content: unknown): string | undefined;
|
|
1787
|
+
export declare function ingestEntries(store: IngestStore, entries: readonly IngestibleEntry[], opts?: {
|
|
1788
|
+
externalize?: {
|
|
1789
|
+
largeFileChars: number;
|
|
1790
|
+
};
|
|
1791
|
+
}): IngestStats;
|
|
1792
|
+
//#endregion
|
|
1793
|
+
//#region src/overflow-window.d.ts
|
|
1794
|
+
/** A window outside this band is a misread rather than a model: no provider
|
|
1795
|
+
* serves a context under a thousand tokens or over a hundred million, and a
|
|
1796
|
+
* number outside it means a pattern caught the request count, a status code, or
|
|
1797
|
+
* something else that was not a limit. A refused parse is the floor path, which
|
|
1798
|
+
* costs one conservative turn instead of a wrong permanent window. */
|
|
1799
|
+
export declare const WINDOW_MIN = 1000;
|
|
1800
|
+
export declare const WINDOW_MAX = 100000000;
|
|
1801
|
+
/** The window in tokens a provider stated in an overflow message, or null when
|
|
1802
|
+
* the message does not name one. Null is an answer: the caller keeps Pi's belief
|
|
1803
|
+
* and arms the one-shot floor, which is all a body like "400/413 status code (no
|
|
1804
|
+
* body)" can support. */
|
|
1805
|
+
export declare function windowFromError(errorMessage: string | undefined): number | null;
|
|
1806
|
+
export declare const WINDOW_PATTERN_EXAMPLES: readonly string[];
|
|
1807
|
+
//#endregion
|
|
1808
|
+
//#region src/recall-view.d.ts
|
|
1809
|
+
export declare const GREP_PAGE_HITS = 20;
|
|
1810
|
+
export type ViewResult = {
|
|
1811
|
+
ok: true;
|
|
1812
|
+
text: string;
|
|
1813
|
+
details: Record<string, unknown>;
|
|
1814
|
+
} | {
|
|
1815
|
+
ok: false;
|
|
1816
|
+
text: string;
|
|
1817
|
+
};
|
|
1818
|
+
export declare function grepView(store: LcmStore, redact: (text: string) => string, params: {
|
|
1819
|
+
query?: string;
|
|
1820
|
+
pattern?: string;
|
|
1821
|
+
caseSensitive?: boolean;
|
|
1822
|
+
offset?: number;
|
|
1823
|
+
limit?: number;
|
|
1824
|
+
summaryId?: number;
|
|
1825
|
+
includeRemoved?: boolean;
|
|
1826
|
+
/** Resume a pattern scan after this entry id, which is what a deadline stop
|
|
1827
|
+
* names. A page cannot express where a clock ran out. */
|
|
1828
|
+
after?: string;
|
|
1829
|
+
now?: () => number;
|
|
1830
|
+
}): ViewResult;
|
|
1831
|
+
export declare function describeView(store: LcmStore, redact: (text: string) => string, id: number): ViewResult;
|
|
1832
|
+
/** List the summary's messages, or read one of them as a window of characters.
|
|
1833
|
+
* A message read without a summary names an entry directly, which is how an
|
|
1834
|
+
* entry that no leaf covers yet is read at all. */
|
|
1835
|
+
export type ExpandParams = {
|
|
1836
|
+
mode: "list";
|
|
1837
|
+
id: number;
|
|
1838
|
+
offset: number;
|
|
1839
|
+
limit: number;
|
|
1840
|
+
maxChars: number;
|
|
1841
|
+
} | {
|
|
1842
|
+
mode: "message";
|
|
1843
|
+
id?: number;
|
|
1844
|
+
entryId: string;
|
|
1845
|
+
charOffset: number;
|
|
1846
|
+
maxChars: number;
|
|
1847
|
+
};
|
|
1848
|
+
export declare function expandView(store: LcmStore, redact: (text: string) => string, params: ExpandParams): ViewResult;
|
|
1849
|
+
export declare function historyManifest(store: LcmStore, limit: number): string;
|
|
1850
|
+
//#endregion
|
|
1851
|
+
//#region src/redact.d.ts
|
|
1852
|
+
/** Redactor that re-reads the flag on every call, so a cockpit toggle takes
|
|
1853
|
+
* effect immediately: a getter, because patchConfig replaces the config object
|
|
1854
|
+
* and closing over it would pin the session-start value. */
|
|
1855
|
+
export declare function makeRedact(getConfig: () => {
|
|
1856
|
+
redactSecrets?: boolean;
|
|
1857
|
+
}): (text: string) => string;
|
|
1858
|
+
export declare function redactSecrets(text: string): string;
|
|
1859
|
+
//#endregion
|
|
1860
|
+
//#region src/retention.d.ts
|
|
1861
|
+
export declare const SCRATCH_MAX_AGE_MS: number;
|
|
1862
|
+
export interface DbFootprint {
|
|
1863
|
+
path: string;
|
|
1864
|
+
name: string;
|
|
1865
|
+
sizeBytes: number;
|
|
1866
|
+
modifiedAt: number;
|
|
1867
|
+
}
|
|
1868
|
+
export interface GcCandidate {
|
|
1869
|
+
path: string;
|
|
1870
|
+
sizeBytes: number;
|
|
1871
|
+
modifiedAt: number;
|
|
1872
|
+
reason: "session-gone" | "scratch-expired";
|
|
1873
|
+
}
|
|
1874
|
+
export declare function hashSessionPath(sessionFilePath: string): string;
|
|
1875
|
+
export declare function listDbFootprints(dir: string): DbFootprint[];
|
|
1876
|
+
export declare function retentionCandidates(opts: {
|
|
1877
|
+
dir: string;
|
|
1878
|
+
liveDbPath: string | undefined;
|
|
1879
|
+
sessionPathHashes: ReadonlySet<string>;
|
|
1880
|
+
now: number;
|
|
1881
|
+
scratchMaxAgeMs: number;
|
|
1882
|
+
}): GcCandidate[];
|
|
1883
|
+
//#endregion
|
|
1884
|
+
//#region src/retrieval.d.ts
|
|
1885
|
+
export declare const RETRIEVAL_STEPS = 6;
|
|
1886
|
+
export declare const MANIFEST_NODES = 20;
|
|
1887
|
+
export declare const MAX_EXPAND_CHARS = 20000;
|
|
1888
|
+
export declare const DEFAULT_RETRIEVAL_TOKENS = 10000;
|
|
1889
|
+
export declare const MIN_RETRIEVAL_TOKENS = 500;
|
|
1890
|
+
export declare const MAX_RETRIEVAL_TOKENS = 40000;
|
|
1891
|
+
export interface RetrievalRequest {
|
|
1892
|
+
query: string;
|
|
1893
|
+
prompt: string;
|
|
1894
|
+
budgetTokens: number;
|
|
1895
|
+
maxSteps: number;
|
|
1896
|
+
/** Masks stored text on its way to the reader and on its way back. Applied
|
|
1897
|
+
* inside this module so a reader cannot be wired without one. */
|
|
1898
|
+
redact: (text: string) => string;
|
|
1899
|
+
scope?: RetrievalScope;
|
|
1900
|
+
}
|
|
1901
|
+
export type RetrievalScope = {
|
|
1902
|
+
kind: "summary";
|
|
1903
|
+
summaryId: number;
|
|
1904
|
+
} | {
|
|
1905
|
+
kind: "entry";
|
|
1906
|
+
entryId: string;
|
|
1907
|
+
};
|
|
1908
|
+
interface RetrievalBase {
|
|
1909
|
+
steps: number;
|
|
1910
|
+
retrievedTokens: number;
|
|
1911
|
+
usage?: Usage;
|
|
1912
|
+
}
|
|
1913
|
+
export type RetrievalRun = (RetrievalBase & {
|
|
1914
|
+
kind: "answered";
|
|
1915
|
+
text: string;
|
|
1916
|
+
}) | (RetrievalBase & {
|
|
1917
|
+
kind: "stopped";
|
|
1918
|
+
reason: "steps" | "budget";
|
|
1919
|
+
text: string;
|
|
1920
|
+
}) |
|
|
1921
|
+
/** The address named something the store does not hold, so no model was called. */
|
|
1922
|
+
(RetrievalBase & {
|
|
1923
|
+
kind: "refused";
|
|
1924
|
+
reason: string;
|
|
1925
|
+
}) | (RetrievalBase & {
|
|
1926
|
+
kind: "failed";
|
|
1927
|
+
reason: string;
|
|
1928
|
+
});
|
|
1929
|
+
export declare function runRetrieval(store: LcmStore, request: RetrievalRequest, reader: Reader): Promise<RetrievalRun>;
|
|
1930
|
+
//#endregion
|
|
1931
|
+
//#region src/sessions.d.ts
|
|
1932
|
+
/** Most recent past sessions one scan considers. A cap, not a promise: the
|
|
1933
|
+
* deadline below is what bounds the work. */
|
|
1934
|
+
export declare const SESSION_SCAN_LIMIT = 20;
|
|
1935
|
+
/** Session headers one listing reads before it picks the newest `SESSION_SCAN_LIMIT`.
|
|
1936
|
+
* A header is one small read, so a few hundred sessions cost one syscall each. */
|
|
1937
|
+
export declare const SESSION_HEADER_CAP = 200;
|
|
1938
|
+
/** Wall-clock budget for one cross-session search. A scan that runs out returns
|
|
1939
|
+
* what it found and says it was truncated. */
|
|
1940
|
+
export declare const SESSION_SEARCH_DEADLINE_MS = 1500;
|
|
1941
|
+
export declare const SESSION_HITS_LIMIT = 40;
|
|
1942
|
+
export interface SessionRef {
|
|
1943
|
+
hash: string;
|
|
1944
|
+
sessionFile: string;
|
|
1945
|
+
storePath: string;
|
|
1946
|
+
cwd: string;
|
|
1947
|
+
sessionId: string;
|
|
1948
|
+
/** Header timestamp in ms, or the file's mtime when the header has none. */
|
|
1949
|
+
when: number;
|
|
1950
|
+
}
|
|
1951
|
+
/** Why a candidate past session could not be searched. Each reason is a count the
|
|
1952
|
+
* caller reports, never an error it raises: a store this build cannot read is
|
|
1953
|
+
* skipped, not migrated. */
|
|
1954
|
+
export type SessionSkip = "missing" | "unreadable" | "older-generation";
|
|
1955
|
+
/** Line 1 of a session file: `{"type":"session","id":...,"timestamp":...,
|
|
1956
|
+
* "cwd":...}`. Reading stops at the first newline, so a session with a broken
|
|
1957
|
+
* line 2 still lists. */
|
|
1958
|
+
declare const HEADER_LINE: Type.TObject<{
|
|
1959
|
+
type: Type.TLiteral<"session">;
|
|
1960
|
+
cwd: Type.TString;
|
|
1961
|
+
id: Type.TString;
|
|
1962
|
+
timestamp: Type.TString;
|
|
1963
|
+
}>;
|
|
1964
|
+
export declare function readSessionHeader(sessionFile: string): Omit<Static<typeof HEADER_LINE>, "type"> | undefined;
|
|
1965
|
+
export declare function sessionStorePath(storesDir: string, sessionFile: string): string;
|
|
1966
|
+
export declare function listSessions(opts: {
|
|
1967
|
+
sessionsRoot: string;
|
|
1968
|
+
storesDir: string;
|
|
1969
|
+
cwd?: string;
|
|
1970
|
+
limit?: number;
|
|
1971
|
+
/** How deep to walk. Pi nests one directory per project cwd, so 4 is slack. */
|
|
1972
|
+
maxDepth?: number;
|
|
1973
|
+
excludeHash?: string;
|
|
1974
|
+
}): SessionRef[];
|
|
1975
|
+
/** The generation a store file carries, read without migrating it. `undefined`
|
|
1976
|
+
* when the file cannot be read at all. */
|
|
1977
|
+
export declare function storeGeneration(storePath: string): number | undefined;
|
|
1978
|
+
export declare function openSessionStore(ref: SessionRef): {
|
|
1979
|
+
kind: "opened";
|
|
1980
|
+
store: LcmStore;
|
|
1981
|
+
} | {
|
|
1982
|
+
kind: "skipped";
|
|
1983
|
+
reason: SessionSkip;
|
|
1984
|
+
};
|
|
1985
|
+
export interface SessionHit extends GrepHit {
|
|
1986
|
+
session: string;
|
|
1987
|
+
sessionId: string;
|
|
1988
|
+
cwd: string;
|
|
1989
|
+
when: number;
|
|
1990
|
+
}
|
|
1991
|
+
export interface SessionSearchResult {
|
|
1992
|
+
hits: SessionHit[];
|
|
1993
|
+
scanned: number;
|
|
1994
|
+
skipped: Record<SessionSkip, number>;
|
|
1995
|
+
/** True when the deadline or the hit cap ended the scan early. */
|
|
1996
|
+
truncated: boolean;
|
|
1997
|
+
elapsedMs: number;
|
|
1998
|
+
}
|
|
1999
|
+
export declare function searchSessions(refs: readonly SessionRef[], query: string, opts?: {
|
|
2000
|
+
limit?: number;
|
|
2001
|
+
deadlineMs?: number;
|
|
2002
|
+
perSession?: number;
|
|
2003
|
+
now?: () => number;
|
|
2004
|
+
}): SessionSearchResult;
|
|
2005
|
+
export declare const SEARCH_SCOPE_HELP = "session (default) searches this session. sessions searches past sessions in this project's working directory. all_sessions searches past sessions in every project. A cross-session hit is a pointer labelled [lcm:session <hash>] that lcm_expand_query({session, entry_id}) reads, so a past session's text is never injected into the context on its own.";
|
|
2006
|
+
/** The cross-session hits as one labelled block, plus what the scan did not read.
|
|
2007
|
+
* `redact` is required because this hands a past session's stored text to a
|
|
2008
|
+
* model, and that store holds secrets this session's mask has to reach. */
|
|
2009
|
+
export declare function formatSessionHits(result: SessionSearchResult, redact: (text: string) => string): string;
|
|
2010
|
+
export declare function sessionsForScope(opts: {
|
|
2011
|
+
scope: "sessions" | "all_sessions";
|
|
2012
|
+
sessionsRoot: string;
|
|
2013
|
+
storesDir: string;
|
|
2014
|
+
cwd: string;
|
|
2015
|
+
excludeHash?: string;
|
|
2016
|
+
limit?: number;
|
|
2017
|
+
}): SessionRef[];
|
|
2018
|
+
//#endregion
|
|
2019
|
+
//# sourceMappingURL=index.d.mts.map
|