@niwa-labs/digest 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Niwa Labs contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,3 @@
1
- # Temporary Holding Version
1
+ # @niwa-labs/digest
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Pure functions that compress a transcript in the shared session model: full, digest and brief modes, centre truncation, a file index and locators back to the original entries. It contains no model calls.
package/package.json CHANGED
@@ -1,6 +1,13 @@
1
1
  {
2
2
  "name": "@niwa-labs/digest",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.0.0",
4
+ "license": "MIT",
5
+ "type": "module",
6
+ "exports": {
7
+ ".": "./src/index.ts"
8
+ },
9
+ "dependencies": {
10
+ "@niwa-labs/harness": "0.0.0",
11
+ "zod": "^4.6.7"
12
+ }
13
+ }
@@ -0,0 +1,370 @@
1
+ /**
2
+ * The compressor: a reading of a session in the common model, kept whole (`full`), cut to what a
3
+ * reader needs line by line (`digest`), or summed up (`brief`). It is mechanical: no model reads
4
+ * anything here, and the same records always give the same result.
5
+ */
6
+
7
+ import { createHash } from "node:crypto";
8
+
9
+ import { checklistProgress } from "@niwa-labs/harness";
10
+ import type { ImageMark, InsertedRange, SessionRecord, ToolCategory } from "@niwa-labs/harness";
11
+
12
+ import type { Brief, CallStatus, DigestEntry, DigestMode, DigestView, Source } from "./entries.ts";
13
+ import { FileIndexer } from "./files.ts";
14
+ import type { FileIndex } from "./files.ts";
15
+ import { cutEnd, cutMiddle } from "./truncate.ts";
16
+
17
+ /** How long the arguments of a call may be in a digest. */
18
+ export const ARGS_CHARS = 220;
19
+
20
+ /** How much of a call's result a digest keeps by default, the marker of the cut among it. */
21
+ export const RESULT_CHARS = 1_500;
22
+
23
+ /** What a compression is told beside the mode. */
24
+ export interface CompressOptions {
25
+ /** The file index of what was read before, carried on. */
26
+ readonly files?: FileIndex;
27
+ /** How much of a call's result a digest keeps. */
28
+ readonly resultChars?: number;
29
+ /**
30
+ * Whether the reading goes on past these records, so the last turn may not be over. A brief then
31
+ * gives the agent's last message as `tentative`, not as the answer of the turn.
32
+ */
33
+ readonly more?: boolean;
34
+ }
35
+
36
+ /** A compressed reading, and the file index as it stands after it. */
37
+ export interface Compressed {
38
+ readonly view: DigestView;
39
+ readonly files: FileIndex;
40
+ /** How many records left nothing in the view: a harness's own insertion, a record niwa does not know. */
41
+ readonly omitted: number;
42
+ /**
43
+ * Whether the reading started over: it was asked to go on from a locator of another source, and
44
+ * what it holds is the session from its beginning. What was gathered before it no longer holds.
45
+ */
46
+ readonly restarted: boolean;
47
+ }
48
+
49
+ /** A value as JSON with its keys sorted and its undefined members left out: the same value always reads the same. */
50
+ function canonical(value: unknown): string {
51
+ if (Array.isArray(value)) return `[${value.map((each) => (each === undefined ? "null" : canonical(each))).join(",")}]`;
52
+ if (typeof value === "object" && value !== null) {
53
+ const members = Object.entries(value)
54
+ .filter(([, each]) => each !== undefined)
55
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
56
+ .map(([key, each]) => `${JSON.stringify(key)}:${canonical(each)}`);
57
+ return `{${members.join(",")}}`;
58
+ }
59
+ return JSON.stringify(value) ?? "null";
60
+ }
61
+
62
+ /**
63
+ * The source of an entry: the record's locator and a digest of the record. The digest is of the
64
+ * record as the common model says it, its keys in order and without the harness's raw data, so the
65
+ * same record read again through `session.read` has the same digest.
66
+ */
67
+ export function sourceOf(record: SessionRecord): Source {
68
+ const { raw: _raw, ...said } = record as SessionRecord & { raw?: unknown };
69
+ const digest = createHash("sha256").update(canonical(said)).digest("hex").slice(0, 12);
70
+ return { locator: record.locator, digest };
71
+ }
72
+
73
+ /**
74
+ * The tags the harnesses niwa knows wrap their own insertions in: a reminder, the environment, a
75
+ * slash command and what it ran. A tag a person writes (`<user-card>`) is not among them.
76
+ */
77
+ const INSERTION_TAGS = [
78
+ "system-reminder",
79
+ "environment_context",
80
+ "local-command-caveat",
81
+ "local-command-stdout",
82
+ "local-command-stderr",
83
+ "agent-context",
84
+ "command-name",
85
+ "command-message",
86
+ "command-args",
87
+ "command-contents",
88
+ "user-prompt-submit-hook",
89
+ "user_instructions",
90
+ "turn_aborted",
91
+ ];
92
+
93
+ const INSERTION = new RegExp(`<(${INSERTION_TAGS.join("|")})\\b[^>]*>[\\s\\S]*?</\\1>`, "g");
94
+
95
+ /**
96
+ * The text a harness wrapped around what the person said is not what the person said: a block in
97
+ * one of the tags of `INSERTION_TAGS`. What is left is cut of the blank lines the block stood in; a
98
+ * message without an insertion is returned as it was written.
99
+ */
100
+ export function withoutInsertions(text: string): string {
101
+ const said = text.replace(INSERTION, "");
102
+ return said === text ? text : said.replace(/\n{3,}/g, "\n\n").trim();
103
+ }
104
+
105
+ /** What stands in a digest in place of an image. */
106
+ export function imageMarker(mark: ImageMark): string {
107
+ return mark.mime === undefined ? "[image]" : `[image ${mark.mime}]`;
108
+ }
109
+
110
+ /** The stretches of `text` that sit in a tag of `INSERTION_TAGS`. */
111
+ function taggedRanges(text: string): Array<{ from: number; to: number }> {
112
+ return [...text.matchAll(INSERTION)].map((match) => ({ from: match.index, to: match.index + match[0].length }));
113
+ }
114
+
115
+ /** `text` with the ranges `cuts` left out and a marker for each image where its offset says; the rest is kept as it is. */
116
+ function weave(text: string, cuts: ReadonlyArray<{ from: number; to: number }>, images: readonly ImageMark[]): string {
117
+ const merged: Array<{ from: number; to: number }> = [];
118
+ for (const cut of [...cuts].sort((a, b) => a.from - b.from)) {
119
+ const last = merged[merged.length - 1];
120
+ if (last !== undefined && cut.from <= last.to) last.to = Math.max(last.to, cut.to);
121
+ else merged.push({ ...cut });
122
+ }
123
+ /** The text from `from` up to `to` that no cut covers. */
124
+ const kept = (from: number, to: number): string => {
125
+ let out = "";
126
+ let cursor = from;
127
+ for (const cut of merged) {
128
+ if (cut.to <= cursor) continue;
129
+ if (cut.from >= to) break;
130
+ if (cut.from > cursor) out += text.slice(cursor, cut.from);
131
+ cursor = Math.max(cursor, cut.to);
132
+ }
133
+ return cursor < to ? out + text.slice(cursor, to) : out;
134
+ };
135
+ let out = "";
136
+ let position = 0;
137
+ for (const mark of [...images].sort((a, b) => a.at - b.at)) {
138
+ const at = Math.min(Math.max(mark.at, position), text.length);
139
+ out += kept(position, at) + imageMarker(mark);
140
+ position = at;
141
+ }
142
+ return out + kept(position, text.length);
143
+ }
144
+
145
+ /**
146
+ * What a person said in a message, as a digest tells it: what the reader marked as inserted is cut
147
+ * out, an image is a marker where it stood, and every other character is kept as it was written.
148
+ * A message the reader did not mark (`inserted` absent) is cut by the tags of the harnesses niwa
149
+ * knows; one it marked, even with nothing marked, is never cut by a tag, so a person's own
150
+ * `<system-reminder>` stays.
151
+ *
152
+ * `undefined` when nothing but whitespace is left and no image stood in the message.
153
+ */
154
+ export function saidOf(record: {
155
+ readonly text: string;
156
+ readonly inserted?: readonly InsertedRange[] | undefined;
157
+ readonly images?: readonly ImageMark[] | undefined;
158
+ }): string | undefined {
159
+ const images = record.images ?? [];
160
+ if (record.inserted === undefined && images.length === 0) {
161
+ const said = withoutInsertions(record.text);
162
+ return said.length === 0 ? undefined : said;
163
+ }
164
+ const cuts = record.inserted ?? taggedRanges(record.text);
165
+ const said = weave(record.text, cuts, images);
166
+ return images.length === 0 && said.trim().length === 0 ? undefined : said;
167
+ }
168
+
169
+ /** The text of a value a tool gave or was given: a string as it is, anything else as JSON. */
170
+ function textOf(value: unknown): string {
171
+ if (value === undefined) return "";
172
+ return typeof value === "string" ? value : (JSON.stringify(value) ?? "");
173
+ }
174
+
175
+ /**
176
+ * The output of a tool with a marker where each image stood: at its index among the blocks of a
177
+ * list, at its offset in a text, at the end of anything else.
178
+ */
179
+ function withImages(output: unknown, images: readonly ImageMark[] | undefined): unknown {
180
+ if (images === undefined || images.length === 0) return output;
181
+ const marks = [...images].sort((a, b) => a.at - b.at);
182
+ if (Array.isArray(output)) {
183
+ const blocks: unknown[] = [...output];
184
+ marks.forEach((mark, placed) => blocks.splice(Math.min(mark.at + placed, blocks.length), 0, imageMarker(mark)));
185
+ return blocks;
186
+ }
187
+ if (typeof output === "string") return weave(output, [], marks);
188
+ const markers = marks.map(imageMarker).join(" ");
189
+ return output === undefined ? markers : `${textOf(output)} ${markers}`;
190
+ }
191
+
192
+ /** The paths an edit call changes: all it names, else its target. */
193
+ export function editedPaths(record: { readonly target?: string | undefined; readonly paths?: readonly string[] | undefined }): readonly string[] {
194
+ return record.paths ?? (record.target === undefined ? [] : [record.target]);
195
+ }
196
+
197
+ function statusOf(ok: boolean | undefined): CallStatus {
198
+ return ok === false ? "failed" : "done";
199
+ }
200
+
201
+ /** Compress records read in the harness's order. */
202
+ export function compress(records: readonly SessionRecord[], mode: DigestMode, options: CompressOptions = {}): Compressed {
203
+ const files = new FileIndexer(options.files);
204
+ const restarted = records.some((record) => record.reanchored === true);
205
+ switch (mode) {
206
+ case "full":
207
+ return { view: { mode, records: [...records] }, files: files.index(), omitted: 0, restarted };
208
+ case "digest": {
209
+ const { entries, omitted } = digestOf(records, files, options.resultChars ?? RESULT_CHARS);
210
+ return { view: { mode, entries }, files: files.index(), omitted, restarted };
211
+ }
212
+ case "brief": {
213
+ const { brief, omitted } = briefOf(records, options.more ?? false);
214
+ return { view: { mode, brief }, files: files.index(), omitted, restarted };
215
+ }
216
+ }
217
+ }
218
+
219
+ function digestOf(records: readonly SessionRecord[], files: FileIndexer, resultChars: number): { entries: DigestEntry[]; omitted: number } {
220
+ const entries: DigestEntry[] = [];
221
+ /** The tool entries whose result has not been read yet, by call id. */
222
+ const open = new Map<string, Extract<DigestEntry, { kind: "tool" }>>();
223
+ let omitted = 0;
224
+ for (const record of records) {
225
+ const source = sourceOf(record);
226
+ switch (record.kind) {
227
+ case "user": {
228
+ const text = saidOf(record);
229
+ if (text === undefined) omitted += 1;
230
+ else entries.push({ kind: "user", text, source });
231
+ break;
232
+ }
233
+ case "assistant":
234
+ entries.push({ kind: "assistant", text: record.text, source });
235
+ break;
236
+ case "thinking":
237
+ entries.push({ kind: "thinking", text: record.text, redacted: record.redacted, source });
238
+ break;
239
+ case "tool-call": {
240
+ const changed = record.category === "edit";
241
+ const paths = editedPaths(record);
242
+ // A call that names several files names each of them, written once and referred to after.
243
+ const target =
244
+ changed && paths.length > 1
245
+ ? paths.map((path) => namedTarget(record.category, path, changed, files)).join(", ")
246
+ : record.target === undefined
247
+ ? undefined
248
+ : namedTarget(record.category, record.target, changed, files);
249
+ const args = record.input === undefined ? undefined : cutEnd(files.refer(textOf(record.input)), ARGS_CHARS);
250
+ const entry: Extract<DigestEntry, { kind: "tool" }> = {
251
+ kind: "tool",
252
+ callId: record.callId,
253
+ category: record.category,
254
+ name: record.name,
255
+ ...(target === undefined ? {} : { target }),
256
+ ...(args === undefined ? {} : { args }),
257
+ status: "pending",
258
+ source,
259
+ };
260
+ entries.push(entry);
261
+ open.set(record.callId, entry);
262
+ break;
263
+ }
264
+ case "tool-result": {
265
+ const output = textOf(withImages(record.output, record.images));
266
+ const result = output.length === 0 ? undefined : cutMiddle(files.refer(output), { keep: resultChars, source: record.locator });
267
+ const call = open.get(record.callId);
268
+ if (call !== undefined) {
269
+ open.delete(record.callId);
270
+ const index = entries.indexOf(call);
271
+ entries[index] = { ...call, status: statusOf(record.ok), ...(result === undefined ? {} : { result }), resultSource: source };
272
+ } else {
273
+ entries.push({ kind: "result", callId: record.callId, status: statusOf(record.ok), ...(result === undefined ? {} : { result }), source });
274
+ }
275
+ break;
276
+ }
277
+ case "compaction":
278
+ entries.push({ kind: "compaction", summary: cutMiddle(record.summary, { keep: resultChars, source: record.locator }), source });
279
+ break;
280
+ case "checklist":
281
+ entries.push({ kind: "checklist", ...checklistProgress(record), items: record.items, source });
282
+ break;
283
+ case "permission":
284
+ entries.push({ kind: "permission", ...(record.tool === undefined ? {} : { tool: record.tool }), outcome: record.outcome, source });
285
+ break;
286
+ case "nested":
287
+ entries.push({ kind: "nested", child: record.child, source });
288
+ break;
289
+ case "unknown":
290
+ omitted += 1;
291
+ break;
292
+ }
293
+ }
294
+ return { entries, omitted };
295
+ }
296
+
297
+ /** The target of a call as a digest names it: a path through the file index, anything else as it is. */
298
+ function namedTarget(category: ToolCategory, target: string, changed: boolean, files: FileIndexer): string {
299
+ const isPath = (category === "read" || category === "edit") && !/[\r\n]/.test(target);
300
+ return isPath ? files.name(target, changed) : files.refer(target);
301
+ }
302
+
303
+ function briefOf(records: readonly SessionRecord[], more: boolean): { brief: Brief; omitted: number } {
304
+ const asked: Brief["asked"] = [];
305
+ const answered: Brief["answered"] = [];
306
+ /** The edit calls, in order, with the path each named. */
307
+ const edits: Array<{ callId: string; path: string }> = [];
308
+ /** The calls whose result says they failed. */
309
+ const refused = new Set<string>();
310
+ const calls: Partial<Record<ToolCategory, number>> = {};
311
+ let failed = 0;
312
+ let checklist: Brief["checklist"];
313
+ let omitted = 0;
314
+ /** The last message of the agent in the turn that is being read. */
315
+ let answer: Brief["answered"][number] | undefined;
316
+ /** What spoke last: the agent, or something that waits for it. */
317
+ let last: "answered" | "working" | "empty" = "empty";
318
+ for (const record of records) {
319
+ switch (record.kind) {
320
+ case "user": {
321
+ const text = saidOf(record);
322
+ if (text === undefined) {
323
+ omitted += 1;
324
+ break;
325
+ }
326
+ if (answer !== undefined) answered.push(answer);
327
+ answer = undefined;
328
+ asked.push({ text, source: sourceOf(record) });
329
+ last = "working";
330
+ break;
331
+ }
332
+ case "assistant":
333
+ answer = { text: record.text, source: sourceOf(record) };
334
+ last = "answered";
335
+ break;
336
+ case "tool-call":
337
+ calls[record.category] = (calls[record.category] ?? 0) + 1;
338
+ if (record.category === "edit") for (const path of editedPaths(record)) edits.push({ callId: record.callId, path });
339
+ last = "working";
340
+ break;
341
+ case "tool-result":
342
+ if (record.ok === false) {
343
+ failed += 1;
344
+ refused.add(record.callId);
345
+ }
346
+ last = "working";
347
+ break;
348
+ case "checklist":
349
+ checklist = { ...checklistProgress(record), items: record.items };
350
+ break;
351
+ case "unknown":
352
+ omitted += 1;
353
+ break;
354
+ case "thinking":
355
+ case "compaction":
356
+ case "permission":
357
+ case "nested":
358
+ break;
359
+ }
360
+ }
361
+ // A turn the reading goes on into is not over: its last message may be followed by more.
362
+ const tentative = more ? answer : undefined;
363
+ if (answer !== undefined && !more) answered.push(answer);
364
+ // What a call that was refused did not change; a call whose result is not read yet may have.
365
+ const changed = [...new Set(edits.filter((edit) => !refused.has(edit.callId)).map((edit) => edit.path))];
366
+ return {
367
+ brief: { asked, answered, changed, calls, failed, ...(tentative === undefined ? {} : { tentative }), ...(checklist === undefined ? {} : { checklist }), outcome: last },
368
+ omitted,
369
+ };
370
+ }
package/src/entries.ts ADDED
@@ -0,0 +1,114 @@
1
+ /**
2
+ * What a compressed reading of a session is made of. Every entry names the record it came from
3
+ * (`source`): its locator, so the whole of it can be read again, and a digest of it, so a reader can
4
+ * tell the record is still the one that was compressed.
5
+ */
6
+
7
+ import { z } from "zod";
8
+
9
+ import { ChecklistItemSchema, LocatorSchema, SessionIdSchema, SessionRecordSchema, ToolCategorySchema } from "@niwa-labs/harness";
10
+
11
+ /** How much of a session a reading keeps. */
12
+ export const DigestModeSchema = z.enum(["full", "digest", "brief"]);
13
+ export type DigestMode = z.output<typeof DigestModeSchema>;
14
+
15
+ /** The record an entry came from. */
16
+ export const SourceSchema = z.object({
17
+ locator: LocatorSchema,
18
+ /** The first twelve hex digits of the SHA-256 of the record as the reading side gave it. */
19
+ digest: z.string().regex(/^[0-9a-f]{12}$/),
20
+ });
21
+ export type Source = z.output<typeof SourceSchema>;
22
+
23
+ /** How a tool call ended: `pending` while its result is not in what was read. */
24
+ export const CallStatusSchema = z.enum(["done", "failed", "pending"]);
25
+ export type CallStatus = z.output<typeof CallStatusSchema>;
26
+
27
+ const ChecklistProgressSchema = z.object({
28
+ done: z.number().int().nonnegative(),
29
+ total: z.number().int().nonnegative(),
30
+ items: z.array(ChecklistItemSchema),
31
+ });
32
+
33
+ /** One entry of a digest. */
34
+ export const DigestEntrySchema = z.discriminatedUnion("kind", [
35
+ /** What the person said, whole, without what the harness wrapped around it. */
36
+ z.object({ kind: z.literal("user"), text: z.string(), source: SourceSchema }),
37
+ /** What the agent said, whole. */
38
+ z.object({ kind: z.literal("assistant"), text: z.string(), source: SourceSchema }),
39
+ /** What the agent thought, whole; a sealed thought has no text. */
40
+ z.object({ kind: z.literal("thinking"), text: z.string(), redacted: z.boolean(), source: SourceSchema }),
41
+ /**
42
+ * A tool call on one line: what it does, what it is about, how it ended, its arguments cut short,
43
+ * and its result cut in the middle. Paths are written once and referred to by the file index after.
44
+ */
45
+ z.object({
46
+ kind: z.literal("tool"),
47
+ callId: z.string().min(1),
48
+ category: ToolCategorySchema,
49
+ name: z.string().min(1),
50
+ target: z.string().optional(),
51
+ args: z.string().optional(),
52
+ status: CallStatusSchema,
53
+ result: z.string().optional(),
54
+ source: SourceSchema,
55
+ /** The record of the result, when it was read. */
56
+ resultSource: SourceSchema.optional(),
57
+ }),
58
+ /** The result of a call made before what was read. */
59
+ z.object({
60
+ kind: z.literal("result"),
61
+ callId: z.string().min(1),
62
+ status: CallStatusSchema,
63
+ result: z.string().optional(),
64
+ source: SourceSchema,
65
+ }),
66
+ z.object({ kind: z.literal("compaction"), summary: z.string(), source: SourceSchema }),
67
+ z.object({ kind: z.literal("checklist"), ...ChecklistProgressSchema.shape, source: SourceSchema }),
68
+ z.object({
69
+ kind: z.literal("permission"),
70
+ tool: z.string().optional(),
71
+ outcome: z.enum(["allowed", "denied", "unanswered"]),
72
+ source: SourceSchema,
73
+ }),
74
+ z.object({ kind: z.literal("nested"), child: SessionIdSchema, source: SourceSchema }),
75
+ ]);
76
+ export type DigestEntry = z.output<typeof DigestEntrySchema>;
77
+
78
+ /** A message with the record it came from. */
79
+ const SaidSchema = z.object({ text: z.string(), source: SourceSchema });
80
+
81
+ /** A session in brief: what was asked, what was answered, what was changed and how far it got. */
82
+ export const BriefSchema = z.object({
83
+ /** The person's messages. */
84
+ asked: z.array(SaidSchema),
85
+ /** The last message of the agent in each turn. */
86
+ answered: z.array(SaidSchema),
87
+ /**
88
+ * The last message of the agent in a turn that goes on past what was read: the turn's answer
89
+ * unless more follows, which the next window shows. Absent when what was read ends a turn.
90
+ */
91
+ tentative: SaidSchema.optional(),
92
+ /** The paths edit calls named, each once, left out when the call's result says it failed. */
93
+ changed: z.array(z.string()),
94
+ /** How many calls of each category were made. */
95
+ calls: z.partialRecord(ToolCategorySchema, z.number().int().positive()),
96
+ /** How many calls failed. */
97
+ failed: z.number().int().nonnegative(),
98
+ /** The agent's checklist as it last stood, when it kept one. */
99
+ checklist: ChecklistProgressSchema.optional(),
100
+ /**
101
+ * Where the session stands: `answered` when the agent spoke last, `working` when a call or a
102
+ * message waits for it, `empty` when nothing was read.
103
+ */
104
+ outcome: z.enum(["answered", "working", "empty"]),
105
+ });
106
+ export type Brief = z.output<typeof BriefSchema>;
107
+
108
+ /** A reading in one of the modes. */
109
+ export const DigestViewSchema = z.discriminatedUnion("mode", [
110
+ z.object({ mode: z.literal("full"), records: z.array(SessionRecordSchema) }),
111
+ z.object({ mode: z.literal("digest"), entries: z.array(DigestEntrySchema) }),
112
+ z.object({ mode: z.literal("brief"), brief: BriefSchema }),
113
+ ]);
114
+ export type DigestView = z.output<typeof DigestViewSchema>;
package/src/files.ts ADDED
@@ -0,0 +1,86 @@
1
+ /**
2
+ * The file index: each path a window's tool calls name is given a short reference the first time
3
+ * it is seen, and every later mention of it is the reference. A path read and edited ten times is
4
+ * written out once.
5
+ *
6
+ * The index is carried from one window of a session to the next, so a reference keeps its meaning
7
+ * across the whole of a reading.
8
+ */
9
+
10
+ import { z } from "zod";
11
+
12
+ /** One path of the index: its reference is `[F<id>]`. */
13
+ export const IndexedFileSchema = z.object({
14
+ id: z.number().int().positive(),
15
+ path: z.string().min(1),
16
+ /** Whether a call that edits files named it. */
17
+ changed: z.boolean(),
18
+ });
19
+ export type IndexedFile = z.output<typeof IndexedFileSchema>;
20
+
21
+ /** The index: each id and each path is in it once, so a reference names one path. */
22
+ export const FileIndexSchema = z
23
+ .object({ files: z.array(IndexedFileSchema) })
24
+ .refine(
25
+ ({ files }) => new Set(files.map((file) => file.id)).size === files.length && new Set(files.map((file) => file.path)).size === files.length,
26
+ { message: "a file index holds each id and each path once" },
27
+ );
28
+ export type FileIndex = z.output<typeof FileIndexSchema>;
29
+
30
+ export const EMPTY_FILE_INDEX: FileIndex = { files: [] };
31
+
32
+ /** The reference of a path in the index. */
33
+ export function fileRef(id: number): string {
34
+ return `[F${String(id)}]`;
35
+ }
36
+
37
+ function escapeRegExp(text: string): string {
38
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
39
+ }
40
+
41
+ /** The index as it grows over a reading: the paths it holds, and the references in a text. */
42
+ export class FileIndexer {
43
+ readonly #files: IndexedFile[];
44
+ readonly #byPath = new Map<string, IndexedFile>();
45
+ /** The id the next path is given: after the highest one in use, so no two paths share a reference. */
46
+ #nextId: number;
47
+
48
+ constructor(from: FileIndex = EMPTY_FILE_INDEX) {
49
+ this.#files = from.files.map((file) => ({ ...file }));
50
+ this.#nextId = this.#files.reduce((highest, file) => Math.max(highest, file.id), 0) + 1;
51
+ for (const file of this.#files) this.#byPath.set(file.path, file);
52
+ }
53
+
54
+ /**
55
+ * The path as a call names it: whole, with its reference, the first time; the reference after.
56
+ * A call that edits marks the path changed.
57
+ */
58
+ name(path: string, changed: boolean): string {
59
+ const known = this.#byPath.get(path);
60
+ if (known !== undefined) {
61
+ if (changed) known.changed = true;
62
+ return fileRef(known.id);
63
+ }
64
+ const file = { id: this.#nextId++, path, changed };
65
+ this.#files.push(file);
66
+ this.#byPath.set(path, file);
67
+ return `${fileRef(file.id)} ${path}`;
68
+ }
69
+
70
+ /**
71
+ * `text` with every mention of a path the index already holds replaced by its reference. A mention
72
+ * is the whole path: not part of a longer word or path. The text is read once, from its start, so
73
+ * a reference put in is never read again, and the longest path wins where two start together.
74
+ */
75
+ refer(text: string): string {
76
+ if (this.#files.length === 0) return text;
77
+ const paths = [...this.#files].sort((a, b) => b.path.length - a.path.length).map((file) => escapeRegExp(file.path));
78
+ const mention = new RegExp(`(?<![\\w./~-])(?:${paths.join("|")})(?![\\w/-]|\\.\\w)`, "g");
79
+ return text.replace(mention, (path) => fileRef(this.#byPath.get(path)!.id));
80
+ }
81
+
82
+ /** The index as it stands. */
83
+ index(): FileIndex {
84
+ return { files: this.#files.map((file) => ({ ...file })) };
85
+ }
86
+ }
package/src/index.ts ADDED
@@ -0,0 +1,20 @@
1
+ // The compressor of session readings: full, digest and brief, over the common model of
2
+ // `@niwa-labs/harness`, with the file index and the windows a reading is made in. No model is called.
3
+
4
+ export { ARGS_CHARS, RESULT_CHARS, compress, editedPaths, imageMarker, saidOf, sourceOf, withoutInsertions } from "./compress.ts";
5
+ export type { CompressOptions, Compressed } from "./compress.ts";
6
+ export {
7
+ BriefSchema,
8
+ CallStatusSchema,
9
+ DigestEntrySchema,
10
+ DigestModeSchema,
11
+ DigestViewSchema,
12
+ SourceSchema,
13
+ } from "./entries.ts";
14
+ export type { Brief, CallStatus, DigestEntry, DigestMode, DigestView, Source } from "./entries.ts";
15
+ export { EMPTY_FILE_INDEX, FileIndexSchema, FileIndexer, IndexedFileSchema, fileRef } from "./files.ts";
16
+ export type { FileIndex, IndexedFile } from "./files.ts";
17
+ export { cutEnd, cutMarker, cutMiddle } from "./truncate.ts";
18
+ export type { CutOptions } from "./truncate.ts";
19
+ export { charsOf, readWindow, tokensOf } from "./window.ts";
20
+ export type { DigestWindow, WindowOptions } from "./window.ts";
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Cutting a long text in the middle.
3
+ *
4
+ * The start of a tool's output says what it did and the end how it ended, so both are kept and the
5
+ * middle is cut, with a marker that says how much was there and where the whole of it is. An output
6
+ * that ends in an error keeps more of its end, where the error is.
7
+ */
8
+
9
+ /** Words that mark the end of an output as a failure: its end then matters more than its start. */
10
+ const FAILURE = /\b(error|exception|failed|fatal|traceback|panic|exit code|errno|denied)\b/i;
11
+
12
+ /** How much of the end of a text is looked at for a failure. */
13
+ const FAILURE_SCAN = 2_048;
14
+
15
+ /** What a cut is told: how many characters to keep, and where the whole text can be read. */
16
+ export interface CutOptions {
17
+ /** The characters kept, the marker among them. */
18
+ readonly keep: number;
19
+ /** Where the whole text is, said in the marker: the locator of the record it came from. */
20
+ readonly source: string;
21
+ }
22
+
23
+ /** The marker that stands for the middle of a text: how long the text was, and where it is whole. */
24
+ export function cutMarker(length: number, kept: number, source: string): string {
25
+ return `…⟨cut ${String(length - kept)} of ${String(length)} chars, whole at ${source}⟩…`;
26
+ }
27
+
28
+ /** The marker when the full one does not fit: how much was cut, and nothing of where; the entry's source says where. */
29
+ function shortMarker(length: number, kept: number): string {
30
+ return `…⟨cut ${String(length - kept)}⟩…`;
31
+ }
32
+
33
+ /**
34
+ * `text` with its middle cut so that it holds at most `keep` characters, the marker among them. A
35
+ * text that fits is returned as it is, and so is one the cut would not make shorter. The cut moves
36
+ * to a line break when one is near, so a kept line is whole. Where the marker with its source does
37
+ * not fit, a short one that only counts stands in; where not even that fits, the text is cut at
38
+ * its end.
39
+ */
40
+ export function cutMiddle(text: string, { keep, source }: CutOptions): string {
41
+ if (text.length <= keep) return text;
42
+ // The longest the marker can be: the count of what is removed has at most the digits of the length.
43
+ const room = keep - cutMarker(text.length, 0, source).length;
44
+ if (room < 16) {
45
+ const short = keep - shortMarker(text.length, 0).length;
46
+ if (short < 1) return cutEnd(text, keep);
47
+ return `${text.slice(0, short)}${shortMarker(text.length, short)}`;
48
+ }
49
+ const failed = FAILURE.test(text.slice(-FAILURE_SCAN));
50
+ let head = Math.floor(room * (failed ? 0.3 : 0.7));
51
+ let tail = room - head;
52
+ const headBreak = text.lastIndexOf("\n", head);
53
+ if (headBreak > head / 2) head = headBreak + 1;
54
+ const tailStart = text.length - tail;
55
+ const tailBreak = text.indexOf("\n", tailStart);
56
+ if (tailBreak !== -1 && tailBreak - tailStart < tail / 2) tail = text.length - (tailBreak + 1);
57
+ const kept = head + tail;
58
+ const cut = `${text.slice(0, head)}${cutMarker(text.length, kept, source)}${text.slice(text.length - tail)}`;
59
+ return cut.length < text.length ? cut : text;
60
+ }
61
+
62
+ /** `text` cut at its end to at most `max` characters, an ellipsis marking the cut. */
63
+ export function cutEnd(text: string, max: number): string {
64
+ return text.length <= max ? text : `${text.slice(0, Math.max(0, max - 1))}…`;
65
+ }
package/src/window.ts ADDED
@@ -0,0 +1,217 @@
1
+ /**
2
+ * Reading a session in windows that fit a budget.
3
+ *
4
+ * A window takes records while their compressed size fits the budget, then says where the next one
5
+ * starts. Records that share a locator are taken together, since a reading resumed from a locator
6
+ * starts after all of them; and a half-written record ends the window before it, since the reading
7
+ * side does not move past it. The file index is carried from window to window.
8
+ */
9
+
10
+ import type { Locator, SessionRecord } from "@niwa-labs/harness";
11
+
12
+ import { ARGS_CHARS, RESULT_CHARS, compress, editedPaths } from "./compress.ts";
13
+ import type { DigestMode, DigestView } from "./entries.ts";
14
+ import type { FileIndex } from "./files.ts";
15
+
16
+ /** About how many characters make a token. */
17
+ const CHARS_PER_TOKEN = 4;
18
+
19
+ /** What an entry costs beside its text: its kind, its status, its source. */
20
+ const ENTRY_CHARS = 60;
21
+
22
+ /** What the marker of an image costs. */
23
+ const MARKER_CHARS = 16;
24
+
25
+ /** What a brief grows by when a call of a category is counted for the first time, or the first failure is. */
26
+ const COUNTER_CHARS = 16;
27
+
28
+ /** The tokens a text is about. */
29
+ export function tokensOf(chars: number): number {
30
+ return Math.ceil(chars / CHARS_PER_TOKEN);
31
+ }
32
+
33
+ /** What a window is told. */
34
+ export interface WindowOptions {
35
+ readonly mode: DigestMode;
36
+ /** About how many tokens the window may hold; a window always holds at least one record. */
37
+ readonly budget: number;
38
+ /** The file index of the windows before, carried on. */
39
+ readonly files?: FileIndex;
40
+ /** How much of a call's result a digest keeps. */
41
+ readonly resultChars?: number;
42
+ }
43
+
44
+ /** One window of a reading. */
45
+ export interface DigestWindow {
46
+ readonly view: DigestView;
47
+ /** Where the next window reads from; `null` when the window read to the end of what is there now. */
48
+ readonly next: Locator | null;
49
+ /**
50
+ * The last record the window holds: where a reading of a session that is still growing goes on
51
+ * from later. `null` when it holds none.
52
+ */
53
+ readonly upto: Locator | null;
54
+ /** The file index after this window: what the next one is given. */
55
+ readonly files: FileIndex;
56
+ /** How many records left nothing in the view. */
57
+ readonly omitted: number;
58
+ /** Whether the reading started over on another source: what it holds is the session from its beginning. */
59
+ readonly restarted: boolean;
60
+ }
61
+
62
+ function textLength(value: unknown): number {
63
+ if (value === undefined) return 0;
64
+ return typeof value === "string" ? value.length : (JSON.stringify(value)?.length ?? 0);
65
+ }
66
+
67
+ /** About how many characters a record takes in a mode, before it is compressed. */
68
+ export function charsOf(record: SessionRecord, mode: DigestMode, resultChars: number = RESULT_CHARS): number {
69
+ if (mode === "full") return JSON.stringify(record).length;
70
+ switch (record.kind) {
71
+ case "user":
72
+ case "assistant":
73
+ return ENTRY_CHARS + record.text.length + (record.kind === "user" ? MARKER_CHARS * (record.images?.length ?? 0) : 0);
74
+ case "thinking":
75
+ return mode === "brief" ? 0 : ENTRY_CHARS + record.text.length;
76
+ case "tool-call": {
77
+ if (mode === "brief") return 0;
78
+ const named = record.paths === undefined ? (record.target?.length ?? 0) : record.paths.reduce((sum, path) => sum + path.length + 2, 0);
79
+ return ENTRY_CHARS + named + Math.min(textLength(record.input), ARGS_CHARS);
80
+ }
81
+ case "tool-result":
82
+ return mode === "brief" ? 0 : ENTRY_CHARS + Math.min(textLength(record.output) + MARKER_CHARS * (record.images?.length ?? 0), resultChars);
83
+ case "compaction":
84
+ return mode === "brief" ? 0 : ENTRY_CHARS + Math.min(record.summary.length, resultChars);
85
+ case "checklist":
86
+ return ENTRY_CHARS + record.items.reduce((sum, item) => sum + item.text.length + 10, 0);
87
+ case "permission":
88
+ case "nested":
89
+ return mode === "brief" ? 0 : ENTRY_CHARS;
90
+ case "unknown":
91
+ return 0;
92
+ }
93
+ }
94
+
95
+ /**
96
+ * What a brief window has grown by, which `charsOf` cannot say of a record alone: a brief holds each
97
+ * path an edit named once and each counter once, so only the first of each costs anything.
98
+ */
99
+ class BriefLedger {
100
+ readonly #paths = new Set<string>();
101
+ readonly #categories = new Set<string>();
102
+ #failed = false;
103
+
104
+ /** The characters a group adds to the brief beyond what its messages take, and the means to account for them. */
105
+ growth(group: readonly SessionRecord[]): { chars: number; commit: () => void } {
106
+ const paths = new Set<string>();
107
+ const categories = new Set<string>();
108
+ let failed = false;
109
+ let chars = 0;
110
+ for (const record of group) {
111
+ if (record.kind === "tool-call") {
112
+ if (!this.#categories.has(record.category) && !categories.has(record.category)) {
113
+ categories.add(record.category);
114
+ chars += COUNTER_CHARS;
115
+ }
116
+ if (record.category === "edit") {
117
+ for (const path of editedPaths(record)) {
118
+ if (this.#paths.has(path) || paths.has(path)) continue;
119
+ paths.add(path);
120
+ chars += path.length + 4;
121
+ }
122
+ }
123
+ } else if (record.kind === "tool-result" && record.ok === false && !this.#failed && !failed) {
124
+ failed = true;
125
+ chars += COUNTER_CHARS;
126
+ }
127
+ }
128
+ return {
129
+ chars,
130
+ commit: () => {
131
+ for (const path of paths) this.#paths.add(path);
132
+ for (const category of categories) this.#categories.add(category);
133
+ if (failed) this.#failed = true;
134
+ },
135
+ };
136
+ }
137
+ }
138
+
139
+ /**
140
+ * The record as a brief can use it: what a brief never reads (the arguments and results of calls,
141
+ * thoughts, summaries, the raw data of what is not known) is dropped once the record is taken, so a
142
+ * window of a long session does not hold it to its end.
143
+ */
144
+ function forBrief(record: SessionRecord): SessionRecord {
145
+ switch (record.kind) {
146
+ case "tool-call": {
147
+ const { input: _input, ...rest } = record;
148
+ return rest;
149
+ }
150
+ case "tool-result": {
151
+ const { output: _output, ...rest } = record;
152
+ return rest;
153
+ }
154
+ case "thinking":
155
+ return { ...record, text: "" };
156
+ case "compaction":
157
+ return { ...record, summary: "" };
158
+ case "unknown":
159
+ return { ...record, raw: undefined };
160
+ default:
161
+ return record;
162
+ }
163
+ }
164
+
165
+ /**
166
+ * Read one window from `records`: those after the locator the previous window gave, in the
167
+ * harness's order. Only what the window takes is read past by one group, and the iteration is
168
+ * closed when the window is full.
169
+ */
170
+ export async function readWindow(records: AsyncIterable<SessionRecord>, options: WindowOptions): Promise<DigestWindow> {
171
+ const resultChars = options.resultChars ?? RESULT_CHARS;
172
+ const taken: SessionRecord[] = [];
173
+ let group: SessionRecord[] = [];
174
+ let used = 0;
175
+ let full = false;
176
+ let upto: Locator | null = null;
177
+ const ledger = new BriefLedger();
178
+
179
+ /** Take the group read so far, if it fits; say whether the window has room for more. */
180
+ const take = (): boolean => {
181
+ if (group.length === 0) return true;
182
+ const growth = options.mode === "brief" ? ledger.growth(group) : { chars: 0, commit: () => undefined };
183
+ const cost = tokensOf(group.reduce((sum, record) => sum + charsOf(record, options.mode, resultChars), growth.chars));
184
+ if (taken.length > 0 && used + cost > options.budget) return false;
185
+ growth.commit();
186
+ taken.push(...(options.mode === "brief" ? group.map(forBrief) : group));
187
+ used += cost;
188
+ upto = group[group.length - 1]!.locator;
189
+ group = [];
190
+ return true;
191
+ };
192
+
193
+ for await (const record of records) {
194
+ if (record.incomplete) {
195
+ // What was read before it is whole, unless it is part of the same record.
196
+ if (group.length > 0 && group[0]!.locator !== record.locator) take();
197
+ full = true;
198
+ break;
199
+ }
200
+ if (group.length > 0 && group[0]!.locator !== record.locator) {
201
+ if (!take()) {
202
+ full = true;
203
+ break;
204
+ }
205
+ }
206
+ group.push(record);
207
+ }
208
+ // The last group read is whole only when the reading ended by itself; it may not fit.
209
+ if (!full && !take()) full = true;
210
+ const exhausted = !full;
211
+ const compressed = compress(taken, options.mode, {
212
+ ...(options.files === undefined ? {} : { files: options.files }),
213
+ resultChars,
214
+ more: !exhausted,
215
+ });
216
+ return { view: compressed.view, next: exhausted ? null : upto, upto, files: compressed.files, omitted: compressed.omitted, restarted: compressed.restarted };
217
+ }
@@ -0,0 +1,348 @@
1
+ import { expect, test } from "bun:test";
2
+
3
+ import { LocatorSchema, harnessId, sessionId } from "@niwa-labs/harness";
4
+ import type { SessionRecord } from "@niwa-labs/harness";
5
+
6
+ import { DigestViewSchema, FileIndexSchema, FileIndexer, compress, cutMiddle, readWindow, sourceOf, withoutInsertions } from "../src/index.ts";
7
+ import type { DigestEntry, DigestView } from "../src/index.ts";
8
+
9
+ const at = (n: number) => LocatorSchema.parse(`L${String(n)}`);
10
+ const base = (n: number) => ({ locator: at(n), incomplete: false });
11
+
12
+ const user = (n: number, text: string): SessionRecord => ({ kind: "user", text, ...base(n) });
13
+ const assistant = (n: number, text: string): SessionRecord => ({ kind: "assistant", text, ...base(n) });
14
+ const call = (n: number, callId: string, category: "shell" | "read" | "edit", target: string, input: unknown): SessionRecord => ({
15
+ kind: "tool-call",
16
+ callId,
17
+ name: category === "shell" ? "Bash" : category === "read" ? "Read" : "Edit",
18
+ category,
19
+ target,
20
+ input: input as never,
21
+ ...base(n),
22
+ });
23
+ const result = (n: number, callId: string, output: string, ok = true): SessionRecord => ({ kind: "tool-result", callId, output, ok, ...base(n) });
24
+
25
+ const SESSION: SessionRecord[] = [
26
+ user(1, "<system-reminder>be brief</system-reminder>\nFix the parser"),
27
+ { kind: "thinking", text: "The parser is in src/parse.ts", redacted: false, ...base(2) },
28
+ call(3, "c1", "read", "/repo/src/parse.ts", { file_path: "/repo/src/parse.ts" }),
29
+ result(4, "c1", "export function parse() {}"),
30
+ call(5, "c2", "edit", "/repo/src/parse.ts", { file_path: "/repo/src/parse.ts", old: "a", new: "b" }),
31
+ result(6, "c2", "ok"),
32
+ call(7, "c3", "shell", "bun test", { command: "bun test" }),
33
+ result(8, "c3", "1 fail\nerror: expected 1", false),
34
+ { kind: "checklist", items: [{ text: "read", state: "done" }, { text: "fix", state: "active" }], ...base(9) },
35
+ assistant(10, "Fixed the parser."),
36
+ { kind: "unknown", raw: { odd: true }, ...base(11) },
37
+ ];
38
+
39
+ const entriesOf = (view: DigestView): DigestEntry[] => (view.mode === "digest" ? view.entries : []);
40
+
41
+ test("full keeps every record as the reading side gave it", () => {
42
+ const { view, omitted } = compress(SESSION, "full");
43
+ expect(view).toEqual({ mode: "full", records: SESSION });
44
+ expect(omitted).toBe(0);
45
+ });
46
+
47
+ test("a digest keeps the messages whole, puts each call on one entry with its result, and drops what it cannot read", () => {
48
+ const { view, files, omitted } = compress(SESSION, "digest");
49
+ expect(DigestViewSchema.parse(view)).toEqual(view);
50
+ const entries = entriesOf(view);
51
+ expect(entries.map((entry) => entry.kind)).toEqual(["user", "thinking", "tool", "tool", "tool", "checklist", "assistant"]);
52
+ // The harness's own insertion is not what the person said.
53
+ expect(entries[0]).toMatchObject({ kind: "user", text: "Fix the parser" });
54
+ // A path is written once, then referred to; the edit marks it changed.
55
+ expect(entries[2]).toMatchObject({ category: "read", target: "[F1] /repo/src/parse.ts", status: "done", result: "export function parse() {}" });
56
+ expect(entries[3]).toMatchObject({ category: "edit", target: "[F1]", status: "done" });
57
+ expect((entries[3] as { args: string }).args).toContain('"file_path":"[F1]"');
58
+ expect(entries[4]).toMatchObject({ category: "shell", target: "bun test", status: "failed" });
59
+ expect(entries[5]).toMatchObject({ kind: "checklist", done: 1, total: 2 });
60
+ expect(files).toEqual({ files: [{ id: 1, path: "/repo/src/parse.ts", changed: true }] });
61
+ // Every entry names its record, and the result its own.
62
+ expect(entries[2]).toMatchObject({ source: sourceOf(SESSION[2]!), resultSource: sourceOf(SESSION[3]!) });
63
+ expect(omitted).toBe(1);
64
+ });
65
+
66
+ test("a digest cuts long arguments at 220 characters and long results in the middle, naming where they are whole", () => {
67
+ const long = "x".repeat(1_000);
68
+ const records = [call(1, "c", "shell", "echo", { command: long }), result(2, "c", `${"head\n".repeat(400)}${"tail\n".repeat(400)}`)];
69
+ const [entry] = entriesOf(compress(records, "digest", { resultChars: 300 }).view) as Array<Extract<DigestEntry, { kind: "tool" }>>;
70
+ expect(entry!.args!.length).toBe(220);
71
+ expect(entry!.result!.startsWith("head\n")).toBe(true);
72
+ expect(entry!.result!.endsWith("tail\n")).toBe(true);
73
+ expect(entry!.result).toContain("whole at L2");
74
+ expect(entry!.result!.length).toBeLessThanOrEqual(300);
75
+ });
76
+
77
+ test("a result that ends in an error keeps more of its end than its start", () => {
78
+ const text = `${"a".repeat(2_000)}\n${"b".repeat(2_000)} error: boom`;
79
+ const cut = cutMiddle(text, { keep: 400, source: "L1" });
80
+ const [head, tail] = cut.split(/…⟨cut \d+ of \d+ chars, whole at L1⟩…/);
81
+ expect(tail!.length).toBeGreaterThan(head!.length);
82
+ expect(cut.endsWith("error: boom")).toBe(true);
83
+ // A text that fits is left whole.
84
+ expect(cutMiddle("short", { keep: 400, source: "L1" })).toBe("short");
85
+ });
86
+
87
+ test("a result whose call came before the reading is an entry of its own", () => {
88
+ const entries = entriesOf(compress([result(9, "early", "done")], "digest").view);
89
+ expect(entries).toEqual([{ kind: "result", callId: "early", status: "done", result: "done", source: sourceOf(result(9, "early", "done")) }]);
90
+ });
91
+
92
+ test("a brief says what was asked and answered, what changed, how many calls of each kind, the last checklist and the outcome", () => {
93
+ const { view } = compress(SESSION, "brief");
94
+ expect(view).toEqual({
95
+ mode: "brief",
96
+ brief: {
97
+ asked: [{ text: "Fix the parser", source: sourceOf(SESSION[0]!) }],
98
+ answered: [{ text: "Fixed the parser.", source: sourceOf(SESSION[9]!) }],
99
+ changed: ["/repo/src/parse.ts"],
100
+ calls: { read: 1, edit: 1, shell: 1 },
101
+ failed: 1,
102
+ checklist: { done: 1, total: 2, items: [{ text: "read", state: "done" }, { text: "fix", state: "active" }] },
103
+ outcome: "answered",
104
+ },
105
+ });
106
+ expect(compress([user(1, "go"), call(2, "c", "shell", "ls", {})], "brief").view).toMatchObject({ brief: { outcome: "working" } });
107
+ expect(compress([], "brief").view).toMatchObject({ brief: { outcome: "empty", asked: [] } });
108
+ });
109
+
110
+ test("only a tag a harness wraps its insertions in is cut from what the person said", () => {
111
+ expect(withoutInsertions("<environment_context>cwd</environment_context>hi <b>there</b>")).toBe("hi <b>there</b>");
112
+ });
113
+
114
+ async function* stream(records: readonly SessionRecord[]): AsyncIterable<SessionRecord> {
115
+ yield* records;
116
+ }
117
+
118
+ test("windows fit the budget, never split records that share a locator, and carry the file index on", async () => {
119
+ const shared: SessionRecord = { kind: "checklist", items: [{ text: "x", state: "pending" }], ...base(3) };
120
+ const records = [
121
+ call(1, "a", "read", "/r/a.ts", {}),
122
+ result(2, "a", "y".repeat(400)),
123
+ call(3, "b", "read", "/r/a.ts", {}),
124
+ shared,
125
+ result(4, "b", "z".repeat(400)),
126
+ ];
127
+ const first = await readWindow(stream(records), { mode: "digest", budget: 150 });
128
+ expect(first.next).toBe(at(2));
129
+ expect(entriesOf(first.view).map((entry) => entry.kind)).toEqual(["tool"]);
130
+
131
+ const rest = records.slice(2);
132
+ const second = await readWindow(stream(rest), { mode: "digest", budget: 20, files: first.files });
133
+ // The call and the checklist share a locator: they are taken together, over the budget.
134
+ expect(entriesOf(second.view).map((entry) => entry.kind)).toEqual(["tool", "checklist"]);
135
+ expect(entriesOf(second.view)[0]).toMatchObject({ target: "[F1]" });
136
+ expect(second.next).toBe(at(3));
137
+
138
+ const last = await readWindow(stream(records.slice(4)), { mode: "digest", budget: 40, files: second.files });
139
+ expect(last.next).toBeNull();
140
+ expect(last.upto).toBe(at(4));
141
+ });
142
+
143
+ test("a window ends before a half-written record and does not move past it", async () => {
144
+ const records = [user(1, "go"), { ...assistant(2, "hal"), incomplete: true }];
145
+ const window = await readWindow(stream(records), { mode: "digest", budget: 10_000 });
146
+ expect(entriesOf(window.view).map((entry) => entry.kind)).toEqual(["user"]);
147
+ expect(window.next).toBe(at(1));
148
+ });
149
+
150
+ test("a nested session and a permission are entries of the digest", () => {
151
+ const child = sessionId(harnessId("h"), "child");
152
+ const entries = entriesOf(
153
+ compress([{ kind: "nested", child, ...base(1) }, { kind: "permission", tool: "Bash", outcome: "denied", ...base(2) }], "digest").view,
154
+ );
155
+ expect(entries).toMatchObject([{ kind: "nested", child }, { kind: "permission", tool: "Bash", outcome: "denied" }]);
156
+ });
157
+
158
+ test("a record read again in another key order, with an absent field or without its raw data, has the same digest", () => {
159
+ const record: SessionRecord = { kind: "tool-call", callId: "c", name: "Bash", category: "shell", target: "ls", input: { b: 1, a: 2 } as never, locator: at(1), incomplete: false };
160
+ const again = { locator: at(1), incomplete: false, input: { a: 2, b: 1 }, target: "ls", category: "shell", name: "Bash", callId: "c", kind: "tool-call", at: undefined } as unknown as SessionRecord;
161
+ expect(sourceOf(again)).toEqual(sourceOf(record));
162
+ const unknown: SessionRecord = { kind: "unknown", raw: { whatever: true }, locator: at(2), incomplete: false };
163
+ expect(sourceOf({ ...unknown, raw: undefined } as unknown as SessionRecord)).toEqual(sourceOf(unknown));
164
+ expect(sourceOf({ ...record, target: "pwd" })).not.toEqual(sourceOf(record));
165
+ });
166
+
167
+ const briefOf = (view: DigestView) => (view.mode === "brief" ? view.brief : undefined);
168
+
169
+ test("a brief window is charged for the paths and counters it grows, and keeps nothing a brief does not use", async () => {
170
+ const records: SessionRecord[] = [];
171
+ for (let n = 0; n < 100; n += 1) {
172
+ records.push(call(2 * n + 1, `c${String(n)}`, "edit", `/repo/src/file-${String(n)}.ts`, { file_path: `/repo/src/file-${String(n)}.ts` }));
173
+ records.push(result(2 * n + 2, `c${String(n)}`, "x".repeat(5_000)));
174
+ }
175
+ const window = await readWindow(stream(records), { mode: "brief", budget: 20 });
176
+ expect(window.next).not.toBeNull();
177
+ const brief = briefOf(window.view)!;
178
+ expect(brief.changed.length).toBeGreaterThan(0);
179
+ expect(brief.changed.length).toBeLessThan(100);
180
+ // Paths already named cost nothing more.
181
+ const again = records.map((record, index) => (record.kind === "tool-call" ? call(index + 1, record.callId, "edit", "/repo/same.ts", {}) : record));
182
+ const same = await readWindow(stream(again), { mode: "brief", budget: 20 });
183
+ expect(same.next).toBeNull();
184
+ expect(briefOf(same.view)!.changed).toEqual(["/repo/same.ts"]);
185
+ });
186
+
187
+ test("a message is cut only of the tags harnesses wrap their insertions in, and is otherwise left as it was written", () => {
188
+ expect(withoutInsertions("Render <user-card>Ada</user-card> here")).toBe("Render <user-card>Ada</user-card> here");
189
+ expect(withoutInsertions(" indented\n\n\n\n code\n")).toBe(" indented\n\n\n\n code\n");
190
+ expect(withoutInsertions("<system-reminder>x</system-reminder>\n\n\n<user-card/>hi")).toBe("<user-card/>hi");
191
+ const only = user(1, "<user-card>Ada</user-card>");
192
+ expect(entriesOf(compress([only], "digest").view)).toMatchObject([{ kind: "user", text: "<user-card>Ada</user-card>" }]);
193
+ expect(briefOf(compress([only], "brief").view)!.asked).toHaveLength(1);
194
+ });
195
+
196
+ test("a path is replaced where the call names it whole, once, and not inside other words or references", () => {
197
+ const files = new FileIndexer();
198
+ files.name("a", false);
199
+ files.name("b", false);
200
+ expect(files.refer("cat data a b")).toBe("cat data [F1] [F2]");
201
+ expect(files.refer("cat a/b a.txt ab -a")).toBe("cat a/b a.txt ab -a");
202
+ const odd = new FileIndexer();
203
+ odd.name("/p/a", false);
204
+ odd.name("F1", false);
205
+ expect(odd.refer("/p/a F1 [F1]")).toBe("[F1] [F2] [[F2]]");
206
+ const index = new FileIndexer();
207
+ index.name("/repo/x.ts", false);
208
+ expect(index.refer('{"file_path":"/repo/x.ts","other":"/repo/x.tsx"}')).toBe('{"file_path":"[F1]","other":"/repo/x.tsx"}');
209
+ });
210
+
211
+ test("a path with spaces is a path of the index", () => {
212
+ const records = [call(1, "r", "read", "docs/Release Notes.md", {}), call(2, "e", "edit", "docs/Release Notes.md", {})];
213
+ const { view, files } = compress(records, "digest");
214
+ expect(entriesOf(view).map((entry) => (entry as { target: string }).target)).toEqual(["[F1] docs/Release Notes.md", "[F1]"]);
215
+ expect(files.files).toEqual([{ id: 1, path: "docs/Release Notes.md", changed: true }]);
216
+ });
217
+
218
+ test("a file index keeps its ids and paths unique, and goes on after the highest id it holds", () => {
219
+ expect(FileIndexSchema.safeParse({ files: [{ id: 1, path: "a", changed: false }, { id: 1, path: "b", changed: false }] }).success).toBe(false);
220
+ expect(FileIndexSchema.safeParse({ files: [{ id: 1, path: "a", changed: false }, { id: 2, path: "a", changed: false }] }).success).toBe(false);
221
+ const files = new FileIndexer({ files: [{ id: 2, path: "/a", changed: false }] });
222
+ expect(files.name("/b", false)).toBe("[F3] /b");
223
+ expect(files.index().files.map((file) => file.id)).toEqual([2, 3]);
224
+ });
225
+
226
+ test("a cut says how much it removed, and fits the size asked for", () => {
227
+ const text = "line of text\n".repeat(40);
228
+ const sizes = [5, 12, 20, 30, 45, 60, 80, 100, 150];
229
+ for (const keep of sizes) {
230
+ const out = cutMiddle(text, { keep, source: "L12345" });
231
+ expect(out.length).toBeLessThanOrEqual(keep);
232
+ const marker = /…⟨cut (\d+)(?: of (\d+) chars, whole at L12345)?⟩…/.exec(out);
233
+ if (marker === null) continue;
234
+ expect(Number(marker[1])).toBe(text.length - out.replace(marker[0], "").length);
235
+ }
236
+ // The marker with its source does not fit in 20, the one that only counts does.
237
+ expect(/cut \d+⟩/.test(cutMiddle(text, { keep: 20, source: "L12345" }))).toBe(true);
238
+ });
239
+
240
+ test("only the changes that were not refused are in a brief", () => {
241
+ const records = [
242
+ call(1, "a", "edit", "/r/refused.ts", {}),
243
+ result(2, "a", "denied", false),
244
+ call(3, "b", "edit", "/r/done.ts", {}),
245
+ result(4, "b", "ok"),
246
+ call(5, "c", "edit", "/r/again.ts", {}),
247
+ result(6, "c", "no", false),
248
+ call(7, "d", "edit", "/r/again.ts", {}),
249
+ call(8, "e", "edit", "/r/waiting.ts", {}),
250
+ ];
251
+ expect(briefOf(compress(records, "brief").view)!.changed).toEqual(["/r/done.ts", "/r/again.ts", "/r/waiting.ts"]);
252
+ });
253
+
254
+ test("a window that ends inside a turn gives its last message as one the turn may still replace", async () => {
255
+ const records = [user(1, "go"), assistant(2, "checking"), call(3, "c", "shell", "ls", {}), result(4, "c", "x"), assistant(5, "fixed")];
256
+ const whole = briefOf((await readWindow(stream(records), { mode: "brief", budget: 10_000 })).view)!;
257
+ expect(whole.answered.map((said) => said.text)).toEqual(["fixed"]);
258
+ expect(whole.tentative).toBeUndefined();
259
+ const first = await readWindow(stream(records), { mode: "brief", budget: 34 });
260
+ expect(first.next).not.toBeNull();
261
+ const brief = briefOf(first.view)!;
262
+ expect(brief.answered).toEqual([]);
263
+ expect(brief.tentative?.text).toBe("checking");
264
+ // A turn that a message from the person closes is answered.
265
+ const closed = briefOf(compress([user(1, "go"), assistant(2, "done"), user(3, "more")], "brief", { more: true }).view)!;
266
+ expect(closed.answered.map((said) => said.text)).toEqual(["done"]);
267
+ expect(closed.tentative).toBeUndefined();
268
+ });
269
+
270
+
271
+ test("a message whose reader marked what was inserted loses exactly those ranges and keeps every other character", () => {
272
+ const text = "<system-reminder>be brief</system-reminder>\n\nFix <user-card>the</user-card> parser";
273
+ const marked: SessionRecord = { kind: "user", text, inserted: [{ from: 0, to: 45, origin: "harness" }], ...base(1) };
274
+ expect(entriesOf(compress([marked], "digest").view)[0]).toMatchObject({ kind: "user", text: "Fix <user-card>the</user-card> parser" });
275
+ // A reader that marked nothing says the whole text was written by the person, even a tag of a harness.
276
+ const typed: SessionRecord = { kind: "user", text: "explain <system-reminder>x</system-reminder> to me", inserted: [], ...base(2) };
277
+ expect(entriesOf(compress([typed], "digest").view)[0]).toMatchObject({ text: "explain <system-reminder>x</system-reminder> to me" });
278
+ // A reader that does not say leaves the tags of the known harnesses to be cut.
279
+ expect(entriesOf(compress([user(3, "<system-reminder>x</system-reminder>hello")], "digest").view)[0]).toMatchObject({ text: "hello" });
280
+ });
281
+
282
+ test("a message that is all insertion leaves nothing, in a digest and in a brief", () => {
283
+ const all: SessionRecord = { kind: "user", text: "preamble", inserted: [{ from: 0, to: 8, origin: "harness" }], ...base(1) };
284
+ expect(compress([all], "digest")).toMatchObject({ view: { entries: [] }, omitted: 1 });
285
+ expect(briefOf(compress([all], "brief").view)?.asked).toEqual([]);
286
+ });
287
+
288
+ test("an image is a marker where it stood, also in a message with no words and in text that had an insertion cut", () => {
289
+ const only: SessionRecord = { kind: "user", text: "", images: [{ at: 0, mime: "image/png" }], ...base(1) };
290
+ const entry = entriesOf(compress([only], "digest").view)[0];
291
+ expect(entry).toMatchObject({ kind: "user", text: "[image image/png]", source: { locator: at(1) } });
292
+ expect(briefOf(compress([only], "brief").view)?.asked).toMatchObject([{ text: "[image image/png]" }]);
293
+
294
+ const mixed: SessionRecord = {
295
+ kind: "user",
296
+ text: "see <agent-context>c</agent-context>this and that",
297
+ inserted: [{ from: 4, to: 36, origin: "harness" }],
298
+ images: [{ at: 4 }, { at: 44 }],
299
+ ...base(2),
300
+ };
301
+ expect(entriesOf(compress([mixed], "digest").view)[0]).toMatchObject({ text: "see [image]this and[image] that" });
302
+ });
303
+
304
+ test("an image in a tool result is a marker among its blocks, and the result is told without its payload", () => {
305
+ const call1: SessionRecord = { kind: "tool-call", callId: "c1", name: "read", category: "read", target: "/a.png", ...base(1) };
306
+ const list: SessionRecord = { kind: "tool-result", callId: "c1", output: [{ type: "text", text: "a" }, { type: "text", text: "b" }], images: [{ at: 1 }], ...base(2) };
307
+ const [tool] = entriesOf(compress([call1, list], "digest").view);
308
+ expect(tool).toMatchObject({ kind: "tool", result: '[{"type":"text","text":"a"},"[image]",{"type":"text","text":"b"}]' });
309
+ const alone: SessionRecord = { kind: "tool-result", callId: "c1", images: [{ at: 0, mime: "image/jpeg" }], ...base(2) };
310
+ expect(entriesOf(compress([call1, alone], "digest").view)[0]).toMatchObject({ result: "[image image/jpeg]" });
311
+ const text: SessionRecord = { kind: "tool-result", callId: "c1", output: "ab", images: [{ at: 1 }], ...base(2) };
312
+ expect(entriesOf(compress([call1, text], "digest").view)[0]).toMatchObject({ result: "a[image]b" });
313
+ });
314
+
315
+ test("a call that names several files is all of them in the digest and in the brief, and counts them in the window", async () => {
316
+ const patch: SessionRecord = { kind: "tool-call", callId: "p1", name: "apply_patch", category: "edit", target: "/r/a.ts", paths: ["/r/a.ts", "/r/b.ts"], ...base(1) };
317
+ const done: SessionRecord = { kind: "tool-result", callId: "p1", output: "ok", ok: true, ...base(2) };
318
+ const { view, files } = compress([patch, done], "digest");
319
+ expect(entriesOf(view)[0]).toMatchObject({ target: "[F1] /r/a.ts, [F2] /r/b.ts" });
320
+ expect(files.files.map((file) => [file.path, file.changed])).toEqual([["/r/a.ts", true], ["/r/b.ts", true]]);
321
+ expect(briefOf(compress([patch, done], "brief").view)?.changed).toEqual(["/r/a.ts", "/r/b.ts"]);
322
+ // A refused patch changed neither.
323
+ expect(briefOf(compress([patch, { ...done, ok: false } as SessionRecord], "brief").view)?.changed).toEqual([]);
324
+ const window = await readWindow(
325
+ (async function* () {
326
+ yield patch;
327
+ yield done;
328
+ })(),
329
+ { mode: "brief", budget: 1_000 },
330
+ );
331
+ expect(briefOf(window.view)?.changed).toEqual(["/r/a.ts", "/r/b.ts"]);
332
+ });
333
+
334
+ test("a window says its reading started over when the first record says so, and a record that goes on does not", async () => {
335
+ const first: SessionRecord = { kind: "assistant", text: "from the top", reanchored: true, ...base(1) };
336
+ const read = async (records: SessionRecord[], mode: "digest" | "brief" | "full") =>
337
+ await readWindow(
338
+ (async function* () {
339
+ yield* records;
340
+ })(),
341
+ { mode, budget: 1_000 },
342
+ );
343
+ for (const mode of ["digest", "brief", "full"] as const) {
344
+ expect((await read([first, assistant(2, "more")], mode)).restarted).toBe(true);
345
+ expect((await read([assistant(2, "more")], mode)).restarted).toBe(false);
346
+ }
347
+ expect(compress([first], "digest").restarted).toBe(true);
348
+ });
package/tsconfig.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "include": [
4
+ "src"
5
+ ]
6
+ }