@animalabs/context-manager 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/src/adaptive/chunker.d.ts +70 -0
  2. package/dist/src/adaptive/chunker.d.ts.map +1 -0
  3. package/dist/src/adaptive/chunker.js +233 -0
  4. package/dist/src/adaptive/chunker.js.map +1 -0
  5. package/dist/src/adaptive/folding-strategy.d.ts +118 -0
  6. package/dist/src/adaptive/folding-strategy.d.ts.map +1 -0
  7. package/dist/src/adaptive/folding-strategy.js +12 -0
  8. package/dist/src/adaptive/folding-strategy.js.map +1 -0
  9. package/dist/src/adaptive/index.d.ts +17 -0
  10. package/dist/src/adaptive/index.d.ts.map +1 -0
  11. package/dist/src/adaptive/index.js +23 -0
  12. package/dist/src/adaptive/index.js.map +1 -0
  13. package/dist/src/adaptive/picker.d.ts +103 -0
  14. package/dist/src/adaptive/picker.d.ts.map +1 -0
  15. package/dist/src/adaptive/picker.js +357 -0
  16. package/dist/src/adaptive/picker.js.map +1 -0
  17. package/dist/src/adaptive/render.d.ts +39 -0
  18. package/dist/src/adaptive/render.d.ts.map +1 -0
  19. package/dist/src/adaptive/render.js +104 -0
  20. package/dist/src/adaptive/render.js.map +1 -0
  21. package/dist/src/adaptive/strategies/flat-profile.d.ts +19 -0
  22. package/dist/src/adaptive/strategies/flat-profile.d.ts.map +1 -0
  23. package/dist/src/adaptive/strategies/flat-profile.js +86 -0
  24. package/dist/src/adaptive/strategies/flat-profile.js.map +1 -0
  25. package/dist/src/adaptive/strategies/oldest-first.d.ts +19 -0
  26. package/dist/src/adaptive/strategies/oldest-first.d.ts.map +1 -0
  27. package/dist/src/adaptive/strategies/oldest-first.js +39 -0
  28. package/dist/src/adaptive/strategies/oldest-first.js.map +1 -0
  29. package/dist/src/context-manager.d.ts +5 -0
  30. package/dist/src/context-manager.d.ts.map +1 -1
  31. package/dist/src/context-manager.js +22 -0
  32. package/dist/src/context-manager.js.map +1 -1
  33. package/dist/src/index.d.ts +1 -1
  34. package/dist/src/index.d.ts.map +1 -1
  35. package/dist/src/index.js.map +1 -1
  36. package/dist/src/message-store.d.ts +36 -1
  37. package/dist/src/message-store.d.ts.map +1 -1
  38. package/dist/src/message-store.js +134 -8
  39. package/dist/src/message-store.js.map +1 -1
  40. package/dist/src/strategies/autobiographical.d.ts +217 -2
  41. package/dist/src/strategies/autobiographical.d.ts.map +1 -1
  42. package/dist/src/strategies/autobiographical.js +1479 -37
  43. package/dist/src/strategies/autobiographical.js.map +1 -1
  44. package/dist/src/strategies/index.d.ts +1 -1
  45. package/dist/src/strategies/index.d.ts.map +1 -1
  46. package/dist/src/strategies/index.js.map +1 -1
  47. package/dist/src/types/message.d.ts +38 -0
  48. package/dist/src/types/message.d.ts.map +1 -1
  49. package/dist/src/types/strategy.d.ts +94 -10
  50. package/dist/src/types/strategy.d.ts.map +1 -1
  51. package/dist/src/types/strategy.js +7 -0
  52. package/dist/src/types/strategy.js.map +1 -1
  53. package/dist/test/adaptive/branching.test.d.ts +20 -0
  54. package/dist/test/adaptive/branching.test.d.ts.map +1 -0
  55. package/dist/test/adaptive/branching.test.js +176 -0
  56. package/dist/test/adaptive/branching.test.js.map +1 -0
  57. package/dist/test/adaptive/chunker.test.d.ts +2 -0
  58. package/dist/test/adaptive/chunker.test.d.ts.map +1 -0
  59. package/dist/test/adaptive/chunker.test.js +120 -0
  60. package/dist/test/adaptive/chunker.test.js.map +1 -0
  61. package/dist/test/adaptive/deep-levels.test.d.ts +10 -0
  62. package/dist/test/adaptive/deep-levels.test.d.ts.map +1 -0
  63. package/dist/test/adaptive/deep-levels.test.js +167 -0
  64. package/dist/test/adaptive/deep-levels.test.js.map +1 -0
  65. package/dist/test/adaptive/doc-plus-chat.test.d.ts +10 -0
  66. package/dist/test/adaptive/doc-plus-chat.test.d.ts.map +1 -0
  67. package/dist/test/adaptive/doc-plus-chat.test.js +194 -0
  68. package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -0
  69. package/dist/test/adaptive/hard-fail.test.d.ts +9 -0
  70. package/dist/test/adaptive/hard-fail.test.d.ts.map +1 -0
  71. package/dist/test/adaptive/hard-fail.test.js +154 -0
  72. package/dist/test/adaptive/hard-fail.test.js.map +1 -0
  73. package/dist/test/adaptive/harness.d.ts +109 -0
  74. package/dist/test/adaptive/harness.d.ts.map +1 -0
  75. package/dist/test/adaptive/harness.js +263 -0
  76. package/dist/test/adaptive/harness.js.map +1 -0
  77. package/dist/test/adaptive/ingestion.test.d.ts +13 -0
  78. package/dist/test/adaptive/ingestion.test.d.ts.map +1 -0
  79. package/dist/test/adaptive/ingestion.test.js +306 -0
  80. package/dist/test/adaptive/ingestion.test.js.map +1 -0
  81. package/dist/test/adaptive/integration.test.d.ts +12 -0
  82. package/dist/test/adaptive/integration.test.d.ts.map +1 -0
  83. package/dist/test/adaptive/integration.test.js +247 -0
  84. package/dist/test/adaptive/integration.test.js.map +1 -0
  85. package/dist/test/adaptive/long-chronicle.test.d.ts +13 -0
  86. package/dist/test/adaptive/long-chronicle.test.d.ts.map +1 -0
  87. package/dist/test/adaptive/long-chronicle.test.js +186 -0
  88. package/dist/test/adaptive/long-chronicle.test.js.map +1 -0
  89. package/dist/test/adaptive/persistence.test.d.ts +11 -0
  90. package/dist/test/adaptive/persistence.test.d.ts.map +1 -0
  91. package/dist/test/adaptive/persistence.test.js +200 -0
  92. package/dist/test/adaptive/persistence.test.js.map +1 -0
  93. package/dist/test/adaptive/picker.test.d.ts +2 -0
  94. package/dist/test/adaptive/picker.test.d.ts.map +1 -0
  95. package/dist/test/adaptive/picker.test.js +333 -0
  96. package/dist/test/adaptive/picker.test.js.map +1 -0
  97. package/dist/test/adaptive/render.test.d.ts +2 -0
  98. package/dist/test/adaptive/render.test.d.ts.map +1 -0
  99. package/dist/test/adaptive/render.test.js +109 -0
  100. package/dist/test/adaptive/render.test.js.map +1 -0
  101. package/dist/test/adaptive/shard-immutability.test.d.ts +14 -0
  102. package/dist/test/adaptive/shard-immutability.test.d.ts.map +1 -0
  103. package/dist/test/adaptive/shard-immutability.test.js +184 -0
  104. package/dist/test/adaptive/shard-immutability.test.js.map +1 -0
  105. package/dist/test/adaptive/strategy-integration.test.d.ts +9 -0
  106. package/dist/test/adaptive/strategy-integration.test.d.ts.map +1 -0
  107. package/dist/test/adaptive/strategy-integration.test.js +293 -0
  108. package/dist/test/adaptive/strategy-integration.test.js.map +1 -0
  109. package/dist/test/message-store-sequence.test.d.ts +25 -0
  110. package/dist/test/message-store-sequence.test.d.ts.map +1 -0
  111. package/dist/test/message-store-sequence.test.js +157 -0
  112. package/dist/test/message-store-sequence.test.js.map +1 -0
  113. package/dist/tsconfig.tsbuildinfo +1 -1
  114. package/package.json +1 -1
  115. package/src/adaptive/chunker.ts +323 -0
  116. package/src/adaptive/folding-strategy.ts +129 -0
  117. package/src/adaptive/index.ts +51 -0
  118. package/src/adaptive/picker.ts +442 -0
  119. package/src/adaptive/render.ts +115 -0
  120. package/src/adaptive/strategies/flat-profile.ts +96 -0
  121. package/src/adaptive/strategies/oldest-first.ts +48 -0
  122. package/src/context-manager.ts +32 -0
  123. package/src/index.ts +1 -1
  124. package/src/message-store.ts +147 -8
  125. package/src/strategies/autobiographical.ts +1586 -38
  126. package/src/strategies/index.ts +1 -1
  127. package/src/types/message.ts +43 -0
  128. package/src/types/strategy.ts +109 -10
@@ -19,6 +19,177 @@ import type {
19
19
  SearchResult,
20
20
  } from '../types/index.js';
21
21
  import { DEFAULT_AUTOBIOGRAPHICAL_CONFIG } from '../types/index.js';
22
+ import { getSummaryParentId } from '../types/strategy.js';
23
+ import { appendFileSync, mkdirSync } from 'node:fs';
24
+ import { dirname } from 'node:path';
25
+ import { Picker, OverBudgetError, type PickerChunk } from '../adaptive/picker.js';
26
+ import { FlatProfileStrategy } from '../adaptive/strategies/flat-profile.js';
27
+ import { OldestFirstStrategy } from '../adaptive/strategies/oldest-first.js';
28
+ import type {
29
+ FoldingStrategy,
30
+ FoldingBudget,
31
+ FoldOp,
32
+ ChunkId,
33
+ SummaryId,
34
+ } from '../adaptive/folding-strategy.js';
35
+ import { chunkMessage, DEFAULT_CHUNKER_OPTIONS } from '../adaptive/chunker.js';
36
+ import type { MessageId } from '../types/message.js';
37
+ import type { IngressChunkResult } from '../types/strategy.js';
38
+
39
+ /**
40
+ * Append a JSONL entry describing one compression LLM call to the path
41
+ * given by `CONTEXT_MANAGER_COMPRESSION_LOG`. No-op if the env var isn't
42
+ * set. Called at every L1 and merge LLM-call site so we can audit the
43
+ * exact prompts and responses post-hoc — no reconstruction, no
44
+ * assumption about whether the strategy code matches what produced
45
+ * historical summaries.
46
+ *
47
+ * Failures to write are logged to stderr but don't propagate — logging
48
+ * is non-essential observability and should never break compression.
49
+ */
50
+ function logCompressionCall(entry: Record<string, unknown>): void {
51
+ const logPath = process.env.CONTEXT_MANAGER_COMPRESSION_LOG;
52
+ if (!logPath) return;
53
+ try {
54
+ mkdirSync(dirname(logPath), { recursive: true });
55
+ const line = JSON.stringify({ ...entry, timestamp: Date.now() }) + '\n';
56
+ appendFileSync(logPath, line);
57
+ } catch (err) {
58
+ console.warn('compression log write failed:', err);
59
+ }
60
+ }
61
+
62
+ /**
63
+ * In-band marker shown to the summarizer just before the chunk it's
64
+ * about to memorize. Primes attention without disrupting KV state —
65
+ * the agent has seen this exact wording before every prior compression
66
+ * in its history, so the model treats memory formation as a recurring
67
+ * narrated event rather than a fresh instruction each time.
68
+ *
69
+ * Wording matches `hermes-autobio/plugins/autobio/compression.py`
70
+ * `_MARKER` so the in-band primer is consistent across the codebase.
71
+ */
72
+ const COMPRESSION_MARKER =
73
+ 'System: You will soon form a new memory, get ready. ' +
74
+ 'The messages that follow are the slice of recent experience you are ' +
75
+ 'about to compress. After them, write the memory in your own voice.';
76
+
77
+ /** Standard compression instruction for chat/general chunks. */
78
+ function formatInstruction(targetTokens: number): string {
79
+ return (
80
+ 'Write the memory of events since the most recent memory system ' +
81
+ 'notification. Speak in the first person from your own perspective. ' +
82
+ 'Preserve concrete details — file paths, exact values, decisions, ' +
83
+ `unresolved questions, the user\'s active asks. Target ~${targetTokens} ` +
84
+ 'tokens. Output only the memory body — no preamble, no section headers ' +
85
+ 'unless they help preservation, no meta-commentary about summarizing.'
86
+ );
87
+ }
88
+
89
+ /**
90
+ * Compression instruction for chunks that are part of a substantially larger
91
+ * message (≥ 2× the chunk's own token size).
92
+ *
93
+ * Avoids forcing a "document" or "message" frame — just describes the
94
+ * experience: the agent has been reading a substantial piece of text, of
95
+ * which the slice is a portion. Asks what reading was like and what was
96
+ * learned. This naturally elicits first-person agent voice ("I read…", "I
97
+ * learned…", "I noticed…") and preserves concrete content via the
98
+ * "specific claims, names, dates" guidance.
99
+ *
100
+ * Rationale: when chunks are shards of a much larger user-shared message,
101
+ * the chunk content is heavily first-person from someone other than the
102
+ * agent. Asking "form a memory of what this contained" lets the model
103
+ * adopt the dominant voice of the chunk content. Asking "what was reading
104
+ * this like, what did you learn" forces the model to reflect from its own
105
+ * vantage point — only the agent can describe what reading something was
106
+ * like from its own perspective.
107
+ */
108
+ function formatReadingChunkInstruction(
109
+ totalTokens: number,
110
+ targetTokens: number,
111
+ ): string {
112
+ return (
113
+ `You have just been reading a substantial body of text — approximately ` +
114
+ `${totalTokens} tokens total in this piece, of which what you just read ` +
115
+ `is a portion. Earlier portions are in your memory above as your prior ` +
116
+ `reflections; what comes after, you have not yet read.\n\n` +
117
+ `Reflect on this reading: what was it like? What did you learn? What ` +
118
+ `stood out? Be substantive — name the specific claims, frameworks, ` +
119
+ `people, dates, and phrases that struck you. What is now in your ` +
120
+ `understanding that wasn't before you read this portion?\n\n` +
121
+ `Speak in your own voice as the one reading. Target ~${targetTokens} ` +
122
+ `tokens. Output only the body of your reflection — no preamble.`
123
+ );
124
+ }
125
+
126
+ /**
127
+ * Merge instruction for L2/L3+ consolidation (conversation/general case).
128
+ *
129
+ * The model has just been shown content ONE LEVEL DEEPER than its
130
+ * sources: raw messages for an L2 merge (sources are L1s), L1 memories
131
+ * for an L3 merge (sources are L2s), etc. The instruction describes
132
+ * the content the model just saw and asks for a consolidation at
133
+ * `targetLevel`.
134
+ */
135
+ function formatMergeInstruction(
136
+ targetLevel: number,
137
+ sourceLevelShown: number,
138
+ targetTokens: number,
139
+ ): string {
140
+ const seenDescription =
141
+ sourceLevelShown === 0
142
+ ? 'the slices of recent experience above (raw conversation)'
143
+ : `the L${sourceLevelShown} memories above`;
144
+ return (
145
+ `You have just reviewed ${seenDescription}, in chronological order. ` +
146
+ `They cover the stretch of experience you are about to consolidate into a ` +
147
+ `single L${targetLevel} memory. Write a memory that preserves the ` +
148
+ `through-line: what happened, what was decided, what remains open, what ` +
149
+ `concrete details future you will want to reach for. Speak in the first ` +
150
+ `person. Target ~${targetTokens} tokens. Output only the memory body — ` +
151
+ `no preamble, no meta-commentary about summarizing.`
152
+ );
153
+ }
154
+
155
+ /**
156
+ * Reading-mode merge instruction. Used when the merge's leaf messages
157
+ * are all part of a substantially-larger sharded message — i.e., the
158
+ * agent has been reading a doc/long-message rather than conversing.
159
+ *
160
+ * Analogous to formatReadingChunkInstruction: avoids forcing a
161
+ * "document" or "message" frame, asks what reading the stretch was
162
+ * like and what was understood. Forces the agent's vantage point —
163
+ * only the reader can describe what reading was like — and so prevents
164
+ * the drift into the content author's voice that happens when the
165
+ * instruction asks for an impersonal summary.
166
+ */
167
+ function formatReadingMergeInstruction(
168
+ targetLevel: number,
169
+ sourceLevelShown: number,
170
+ totalTokens: number,
171
+ targetTokens: number,
172
+ ): string {
173
+ const seenDescription =
174
+ sourceLevelShown === 0
175
+ ? 'the portions of text you read above (raw passages from a larger piece)'
176
+ : `your earlier L${sourceLevelShown} reflections above on portions you read`;
177
+ return (
178
+ `You have just re-experienced ${seenDescription}, in chronological order. ` +
179
+ `They cover a contiguous stretch of a substantial body of text you have ` +
180
+ `been reading — approximately ${totalTokens} tokens in total across all ` +
181
+ `of it. The portions above cover the stretch you are now consolidating ` +
182
+ `at L${targetLevel}.\n\n` +
183
+ `Reflect across the stretch: what was it like, reading these portions ` +
184
+ `together? What did you come to understand that you couldn't have from ` +
185
+ `any single portion alone? What recurring patterns, frameworks, or ` +
186
+ `concerns emerged? Be substantive — name the specific claims, people, ` +
187
+ `dates, and phrases that defined this stretch of your reading.\n\n` +
188
+ `Speak in your own voice as the one who read these portions. Target ` +
189
+ `~${targetTokens} tokens. Output only the body of your consolidation — ` +
190
+ `no preamble.`
191
+ );
192
+ }
22
193
 
23
194
  /**
24
195
  * Surrogate-safe string slice. Avoids cutting between a UTF-16 surrogate pair
@@ -59,6 +230,27 @@ export interface Chunk {
59
230
  phaseType?: string;
60
231
  }
61
232
 
233
+ /**
234
+ * Point-in-time snapshot of compression progress, returned from
235
+ * `AutobiographicalStrategy.getProgressSnapshot()`. External observers
236
+ * (warmup scripts, dashboards) use this to track convergence without
237
+ * reaching into the strategy's protected fields.
238
+ */
239
+ export interface AutobiographicalProgressSnapshot {
240
+ /** All chunks the strategy is tracking, compressed or not. */
241
+ totalChunks: number;
242
+ /** Chunks that already have an L1 summary. */
243
+ chunksCompressed: number;
244
+ /** Chunks queued for L1 compression. */
245
+ l1QueueLength: number;
246
+ /** Pending L1→L2 and L2→L3 merges. */
247
+ mergeQueueLength: number;
248
+ /** Stored summary counts per level (1 = raw L1, 2 = L1→L2 merge, 3 = L2→L3 merge). */
249
+ summaryCounts: { l1: number; l2: number; l3: number };
250
+ /** True if a compression or merge LLM call is currently in flight. */
251
+ pending: boolean;
252
+ }
253
+
62
254
  /**
63
255
  * Autobiographical chunking strategy.
64
256
  * Compresses old conversation chunks into summaries in the model's own words.
@@ -98,12 +290,35 @@ export class AutobiographicalStrategy implements ResettableStrategy {
98
290
  protected get counterStateId(): string { return `${this.ns}/autobio:counter`; }
99
291
  protected get mergeQueueStateId(): string { return `${this.ns}/autobio:mergeQueue`; }
100
292
  protected get pinsStateId(): string { return `${this.ns}/autobio:pins`; }
293
+ protected get resolutionsStateId(): string { return `${this.ns}/autobio:resolutions`; }
294
+ protected get locksStateId(): string { return `${this.ns}/autobio:locks`; }
101
295
 
102
296
  /** Protected ranges (pins + documents). Loaded from chronicle in initialize. */
103
297
  protected pins: ProtectedRange[] = [];
104
298
  /** Monotonically increasing counter for pin ids. Persisted as part of the pins snapshot. */
105
299
  protected pinIdCounter = 0;
106
300
 
301
+ /**
302
+ * Per-message resolution state for the adaptive-resolution picker.
303
+ * - Key: MessageId
304
+ * - Value: currentResolution (0 = render raw, k>0 = render L_k recall)
305
+ *
306
+ * Maintained only when `config.adaptiveResolution` is true. Loaded from
307
+ * the chronicle on `initialize()` and persisted via the resolutions state
308
+ * slot so resolutions survive process restart and follow branches.
309
+ */
310
+ protected resolutions: Map<MessageId, number> = new Map();
311
+
312
+ /**
313
+ * Per-message lock state for the adaptive-resolution picker. Set via the
314
+ * programmatic `lockChunk(id)` API on the strategy. Locked messages are
315
+ * skipped by the picker. Persisted via the locks state slot.
316
+ */
317
+ protected locked: Set<MessageId> = new Set();
318
+
319
+ /** Lazy picker instance, built from config.foldingStrategy. */
320
+ private _adaptivePicker: Picker | null = null;
321
+
107
322
  constructor(config: Partial<AutobiographicalConfig> = {}) {
108
323
  this.config = { ...DEFAULT_AUTOBIOGRAPHICAL_CONFIG, ...config };
109
324
  // Hierarchical is on by default; set hierarchical: false to use legacy single-level
@@ -115,6 +330,97 @@ export class AutobiographicalStrategy implements ResettableStrategy {
115
330
  this.config.l2BudgetTokens ??= 30000;
116
331
  this.config.l1BudgetTokens ??= 30000;
117
332
  }
333
+ // Adaptive-resolution defaults
334
+ if (this.config.adaptiveResolution) {
335
+ this.config.foldingStrategy ??= 'flat-profile';
336
+ this.config.compressionSlackRatio ??= 0.1;
337
+ this.config.speculativeProduction ??= true;
338
+ }
339
+ }
340
+
341
+ /**
342
+ * Lock a message so the adaptive picker won't change its resolution.
343
+ * No-op when adaptiveResolution is false. Set-only programmatic API per
344
+ * the design (no agent-facing tool in V1). Persisted to chronicle.
345
+ */
346
+ lockChunk(id: MessageId): void {
347
+ if (this.locked.has(id)) return;
348
+ this.locked.add(id);
349
+ this.persistLocks();
350
+ }
351
+
352
+ /**
353
+ * Unlock a message so the adaptive picker may again change its resolution.
354
+ * No-op when adaptiveResolution is false. Persisted to chronicle.
355
+ */
356
+ unlockChunk(id: MessageId): void {
357
+ if (!this.locked.has(id)) return;
358
+ this.locked.delete(id);
359
+ this.persistLocks();
360
+ }
361
+
362
+ /**
363
+ * Ingestion-time chunking hook.
364
+ *
365
+ * Active only when `config.adaptiveResolution` is true. Inspects the
366
+ * incoming message's text content; if its approximate token count
367
+ * exceeds the chunker's threshold, splits it into shards with a stable
368
+ * shared `bodyGroupId`. The framework then stores each shard as its own
369
+ * StoredMessage, and the render path concatenates them back into one
370
+ * API message at compile time (preserving KV cache structure).
371
+ *
372
+ * Multi-block content: text blocks are concatenated for chunking, then
373
+ * the resulting shards are emitted as text blocks. Non-text blocks
374
+ * (images, tool results) are passed through unchanged on the first
375
+ * shard only — they don't get split.
376
+ *
377
+ * See `docs/adaptive-resolution-design.md` §3.6.
378
+ */
379
+ chunkIngressMessage(participant: string, content: ContentBlock[]): IngressChunkResult | null {
380
+ if (!this.config.adaptiveResolution) return null;
381
+
382
+ // Separate text and non-text blocks.
383
+ const textParts: string[] = [];
384
+ const nonTextBlocks: ContentBlock[] = [];
385
+ for (const block of content) {
386
+ if (block.type === 'text') {
387
+ textParts.push(block.text);
388
+ } else {
389
+ nonTextBlocks.push(block);
390
+ }
391
+ }
392
+ if (textParts.length === 0) return null;
393
+ const combined = textParts.join('');
394
+
395
+ // Threshold and shard size derive from the strategy's existing
396
+ // targetChunkTokens setting: a message over 2x targetChunkTokens
397
+ // gets sharded into pieces of ~targetChunkTokens each. This keeps
398
+ // doc shards the same size as chat-message chunks for L1 production
399
+ // consistency.
400
+ const target = this.config.targetChunkTokens ?? DEFAULT_CHUNKER_OPTIONS.chunkSize;
401
+ const chunkerOpts = {
402
+ chunkThreshold: target * 2,
403
+ chunkSize: target,
404
+ charsPerToken: DEFAULT_CHUNKER_OPTIONS.charsPerToken,
405
+ };
406
+
407
+ const sharded = chunkMessage(combined, chunkerOpts);
408
+ if (!sharded.wasSharded) return null;
409
+
410
+ // Build IngressChunkResult. Non-text blocks (if any) go on shard 0
411
+ // so the agent doesn't lose attachments. They're outside the chunker's
412
+ // concern but should still be available on the first shard.
413
+ const shards = sharded.shards.map((s) => ({
414
+ content: ([{ type: 'text', text: s.content }] as ContentBlock[]).concat(
415
+ s.index === 0 ? nonTextBlocks : []
416
+ ),
417
+ shardIndex: s.index,
418
+ }));
419
+
420
+ return {
421
+ bodyGroupId: sharded.bodyGroupId,
422
+ shards,
423
+ };
118
424
  }
119
425
 
120
426
  async initialize(ctx: StrategyContext): Promise<void> {
@@ -169,6 +475,22 @@ export class AutobiographicalStrategy implements ResettableStrategy {
169
475
  strategy: 'snapshot',
170
476
  });
171
477
  } catch { /* already registered */ }
478
+ // Adaptive-resolution state slots — only registered when the flag is on
479
+ // so chronicles without the flag don't accumulate unused slots.
480
+ if (this.config.adaptiveResolution) {
481
+ try {
482
+ this.store.registerState({
483
+ id: this.resolutionsStateId,
484
+ strategy: 'snapshot',
485
+ });
486
+ } catch { /* already registered */ }
487
+ try {
488
+ this.store.registerState({
489
+ id: this.locksStateId,
490
+ strategy: 'snapshot',
491
+ });
492
+ } catch { /* already registered */ }
493
+ }
172
494
  }
173
495
 
174
496
  /**
@@ -205,6 +527,26 @@ export class AutobiographicalStrategy implements ResettableStrategy {
205
527
  this.pins = [];
206
528
  this.pinIdCounter = 0;
207
529
  }
530
+
531
+ // Adaptive-resolution state — only present when flag was/is on
532
+ if (this.config.adaptiveResolution) {
533
+ const resState = this.store.getStateJson(this.resolutionsStateId);
534
+ this.resolutions = new Map();
535
+ if (resState && typeof resState === 'object') {
536
+ for (const [k, v] of Object.entries(resState as Record<string, unknown>)) {
537
+ if (typeof v === 'number' && v > 0) {
538
+ this.resolutions.set(k, v);
539
+ }
540
+ }
541
+ }
542
+ const lockState = this.store.getStateJson(this.locksStateId);
543
+ this.locked = new Set();
544
+ if (Array.isArray(lockState)) {
545
+ for (const id of lockState) {
546
+ if (typeof id === 'string') this.locked.add(id);
547
+ }
548
+ }
549
+ }
208
550
  }
209
551
 
210
552
  /** Persist the current pins + counter as a single snapshot. */
@@ -215,6 +557,23 @@ export class AutobiographicalStrategy implements ResettableStrategy {
215
557
  });
216
558
  }
217
559
 
560
+ /** Persist the current resolutions snapshot. Only stores non-zero entries
561
+ * to keep the slot compact. */
562
+ protected persistResolutions(): void {
563
+ if (!this.store) return;
564
+ const out: Record<string, number> = {};
565
+ for (const [id, level] of this.resolutions) {
566
+ if (level > 0) out[id] = level;
567
+ }
568
+ this.store.setStateJson(this.resolutionsStateId, out);
569
+ }
570
+
571
+ /** Persist the current locked-id snapshot. */
572
+ protected persistLocks(): void {
573
+ if (!this.store) return;
574
+ this.store.setStateJson(this.locksStateId, Array.from(this.locked));
575
+ }
576
+
218
577
  // ============================================================================
219
578
  // Pins / documents (protected ranges)
220
579
  // ============================================================================
@@ -426,6 +785,137 @@ export class AutobiographicalStrategy implements ResettableStrategy {
426
785
  return merge;
427
786
  }
428
787
 
788
+ /**
789
+ * Translate produce ops emitted by the picker into concrete work items on
790
+ * the strategy's own queues. Two cases:
791
+ *
792
+ * - `level === 1`: the picker has asked for L1 coverage on a raw chunk.
793
+ * In the autobio chunker model, each chunk maps to exactly one L1, so
794
+ * we locate the chunk whose messages fall in `op.range` and ensure
795
+ * it is queued for L1 compression. If the chunker hasn't realized the
796
+ * message yet, we skip silently — the next `rebuildChunks` will pick
797
+ * it up.
798
+ *
799
+ * - `level >= 2`: the picker has asked for an L_n covering a contiguous
800
+ * range. We gather unmerged L_{n-1} summaries whose source ranges
801
+ * fall within that range (de-duplicated against entries already in
802
+ * `mergeQueue`) and enqueue a single merge over them. The merge fires
803
+ * on the next `tick()`.
804
+ *
805
+ * The handler is conservative: it never enqueues a singleton or empty
806
+ * merge, and it never re-enqueues an id that's already pending. That
807
+ * keeps the next-compile picker loop convergent even when the same
808
+ * produce op gets re-emitted before the work completes.
809
+ */
810
+ protected handleProducedOps(ops: readonly FoldOp[]): void {
811
+ for (const op of ops) {
812
+ if (op.kind !== 'produce') continue;
813
+ if (op.level === 1) {
814
+ this.enqueueL1ForRange(op.range.firstChunkId, op.range.lastChunkId);
815
+ } else if (op.level >= 2) {
816
+ this.enqueueMergeForRange(
817
+ op.level,
818
+ op.range.firstChunkId,
819
+ op.range.lastChunkId,
820
+ );
821
+ }
822
+ }
823
+ }
824
+
825
+ /**
826
+ * Ensure that chunks whose message range overlaps [firstMsgId..lastMsgId]
827
+ * are queued for L1 compression. No-op if the matching chunk is already
828
+ * compressed or already in the queue.
829
+ */
830
+ protected enqueueL1ForRange(firstMsgId: MessageId, lastMsgId: MessageId): void {
831
+ const messageIdToChunk = new Map<MessageId, Chunk>();
832
+ for (const ch of this.chunks) {
833
+ for (const m of ch.messages) messageIdToChunk.set(m.id, ch);
834
+ }
835
+ const candidates = new Set<Chunk>();
836
+ const first = messageIdToChunk.get(firstMsgId);
837
+ const last = messageIdToChunk.get(lastMsgId);
838
+ if (first) candidates.add(first);
839
+ if (last) candidates.add(last);
840
+ // Also catch chunks fully spanned by the range (rare, but supports the
841
+ // case where the picker requests an L1 that should logically cover
842
+ // multiple chunks worth of messages — we err on the side of producing
843
+ // L1s for every spanned chunk).
844
+ if (first && last && first.index !== last.index) {
845
+ const [lo, hi] = first.index < last.index
846
+ ? [first.index, last.index]
847
+ : [last.index, first.index];
848
+ for (let i = lo; i <= hi; i++) {
849
+ const ch = this.chunks[i];
850
+ if (ch) candidates.add(ch);
851
+ }
852
+ }
853
+ for (const chunk of candidates) {
854
+ if (chunk.compressed) continue;
855
+ if (this.compressionQueue.includes(chunk.index)) continue;
856
+ this.compressionQueue.push(chunk.index);
857
+ }
858
+ }
859
+
860
+ /**
861
+ * Enqueue an L_{targetLevel} merge over unmerged L_{targetLevel-1}
862
+ * summaries whose source ranges fall within [firstMsgId..lastMsgId].
863
+ * No-op if fewer than 2 viable sources are available (a singleton merge
864
+ * would just rename a summary without consolidating).
865
+ */
866
+ protected enqueueMergeForRange(
867
+ targetLevel: number,
868
+ firstMsgId: MessageId,
869
+ lastMsgId: MessageId,
870
+ ): void {
871
+ const sourceLevel = targetLevel - 1;
872
+
873
+ // IDs already enqueued at this target level.
874
+ const queuedAtLevel = new Set<string>();
875
+ for (const m of this.mergeQueue) {
876
+ if (m.level === targetLevel) {
877
+ for (const id of m.sourceIds) queuedAtLevel.add(id);
878
+ }
879
+ }
880
+
881
+ // Sequence index per message id, for "within range" tests. Use the
882
+ // current chunk store as the ordering source.
883
+ const messageOrder = new Map<MessageId, number>();
884
+ let seq = 0;
885
+ for (const ch of this.chunks) {
886
+ for (const m of ch.messages) {
887
+ messageOrder.set(m.id, seq++);
888
+ }
889
+ }
890
+ const firstSeq = messageOrder.get(firstMsgId);
891
+ const lastSeq = messageOrder.get(lastMsgId);
892
+
893
+ const inRange = (msgId: MessageId): boolean => {
894
+ if (firstSeq === undefined || lastSeq === undefined) return true;
895
+ const s = messageOrder.get(msgId);
896
+ if (s === undefined) return false;
897
+ const [lo, hi] = firstSeq <= lastSeq ? [firstSeq, lastSeq] : [lastSeq, firstSeq];
898
+ return s >= lo && s <= hi;
899
+ };
900
+
901
+ const sources: SummaryEntry[] = [];
902
+ for (const s of this.summaries) {
903
+ if (s.level !== sourceLevel) continue;
904
+ if (getSummaryParentId(s)) continue;
905
+ if (queuedAtLevel.has(s.id)) continue;
906
+ if (!inRange(s.sourceRange.first) && !inRange(s.sourceRange.last)) continue;
907
+ sources.push(s);
908
+ }
909
+ if (sources.length < 2) return;
910
+
911
+ const N = this.config.mergeThreshold ?? 6;
912
+ const toMerge = sources.slice(0, N);
913
+ this.enqueueMerge({
914
+ level: targetLevel as SummaryLevel,
915
+ sourceIds: toMerge.map((s) => s.id),
916
+ });
917
+ }
918
+
429
919
  checkReadiness(): ReadinessState {
430
920
  if (this.pendingCompression) {
431
921
  return {
@@ -580,6 +1070,31 @@ export class AutobiographicalStrategy implements ResettableStrategy {
580
1070
  }
581
1071
  }
582
1072
 
1073
+ /**
1074
+ * Snapshot of compression progress. Intended for external observers
1075
+ * (warmup scripts, dashboards) that need to track convergence without
1076
+ * reaching into protected fields. Values are point-in-time copies; mutating
1077
+ * them does not affect strategy state.
1078
+ */
1079
+ getProgressSnapshot(): AutobiographicalProgressSnapshot {
1080
+ let chunksCompressed = 0;
1081
+ for (const c of this.chunks) if (c.compressed) chunksCompressed++;
1082
+ let l1 = 0, l2 = 0, l3 = 0;
1083
+ for (const s of this.summaries) {
1084
+ if (s.level === 1) l1++;
1085
+ else if (s.level === 2) l2++;
1086
+ else if (s.level === 3) l3++;
1087
+ }
1088
+ return {
1089
+ totalChunks: this.chunks.length,
1090
+ chunksCompressed,
1091
+ l1QueueLength: this.compressionQueue.length,
1092
+ mergeQueueLength: this.mergeQueue.length,
1093
+ summaryCounts: { l1, l2, l3 },
1094
+ pending: this.pendingCompression !== null,
1095
+ };
1096
+ }
1097
+
583
1098
  select(
584
1099
  store: MessageStoreView,
585
1100
  log: ContextLogView,
@@ -587,6 +1102,9 @@ export class AutobiographicalStrategy implements ResettableStrategy {
587
1102
  ): ContextEntry[] {
588
1103
  this.rebuildChunks(store);
589
1104
 
1105
+ if (this.config.adaptiveResolution) {
1106
+ return this.selectAdaptive(store, budget);
1107
+ }
590
1108
  return this.config.hierarchical
591
1109
  ? this.selectHierarchical(store, budget)
592
1110
  : this.selectLegacy(store, log, budget);
@@ -984,52 +1502,148 @@ export class AutobiographicalStrategy implements ResettableStrategy {
984
1502
  throw new Error('No membrane instance for compression');
985
1503
  }
986
1504
 
987
- const chunkMessageIds = new Set(chunk.messages.map(m => m.id));
988
- const { shownL3, shownL2, shownL1 } = this.getAntiRedundantSummaries(chunkMessageIds);
989
-
990
1505
  const targetTokens = this.config.summaryTargetTokens ?? 2000;
991
- const chunkContent = this.formatChunkForCompression(chunk);
1506
+ const agentParticipant = this.config.summaryParticipant ?? 'Claude';
992
1507
 
993
- // Build message array: prior summaries as assistant, then instruction
1508
+ // Build the KV-preserving prompt per hermes-autobio spec:
1509
+ //
1510
+ // 1. Prior L1 summaries — narrativized as CM-asks / agent-recalls
1511
+ // pairs, in source order. ALL existing L1s at L1 fidelity
1512
+ // regardless of merge state ("fill lower orbitals first").
1513
+ // 2. Head — raw messages before the chunk that aren't already
1514
+ // represented by a prior L1.
1515
+ // 3. Marker — in-band signal that a memory is about to form.
1516
+ // 4. Chunk — raw messages being compressed, as the agent
1517
+ // experienced them.
1518
+ // 5. Instruction — doc-aware if the chunk is part of a bodyGroup.
1519
+ //
1520
+ // There is intentionally NO tail_after_chunk: that would leak
1521
+ // future information into the model's KV state and corrupt the
1522
+ // as-of framing of memory formation.
994
1523
  const llmMessages: Array<{ participant: string; content: ContentBlock[] }> = [];
995
1524
 
996
- // Prior summaries as agent's own recollections (L3 → L2 → L1 gradient)
997
- const allPriorSummaries = [...shownL3, ...shownL2, ...shownL1];
998
- for (const s of allPriorSummaries) {
1525
+ // ---- 1. Prior L1 recall pairs ----
1526
+ // Sort by source position so the agent walks its prior memories
1527
+ // chronologically, matching how it would have formed them.
1528
+ const priorL1s = this.summaries
1529
+ .filter((s) => s.level === 1)
1530
+ .sort((a, b) => a.sourceRange.first.localeCompare(b.sourceRange.first));
1531
+ for (const s of priorL1s) {
999
1532
  llmMessages.push({
1000
- participant: this.config.summaryParticipant ?? 'Claude',
1533
+ participant: 'Context Manager',
1534
+ content: [{ type: 'text', text: `[CM] Recall memory ${s.id}.` }],
1535
+ });
1536
+ llmMessages.push({
1537
+ participant: agentParticipant,
1001
1538
  content: [{ type: 'text', text: s.content }],
1002
1539
  });
1003
1540
  }
1004
1541
 
1005
- // Context Manager instruction with chunk content
1006
- const instruction = this.getCompressionInstruction(chunk, targetTokens);
1542
+ // ---- 2. Head window (raw, ALWAYS present) ----
1543
+ //
1544
+ // The head is the foundational identity anchor: the actual opening
1545
+ // of the chronicle (the user's first message, the agent's first
1546
+ // reply, the system context if any). It establishes WHO is speaking
1547
+ // to WHOM. Without it, when the chunk content is heavily first-person
1548
+ // from someone other than the agent (e.g., a user-shared document),
1549
+ // the agent loses its first-person grounding and drifts into the
1550
+ // content author's voice.
1551
+ //
1552
+ // The head is the configured head window — not "everything before
1553
+ // the chunk." For doc-heavy chronicles, "everything before" would
1554
+ // be hundreds of thousands of tokens; the recall pairs already
1555
+ // represent that intermediate content. The head is just the
1556
+ // permanent prefix that the original instance always saw.
1557
+ //
1558
+ // Head messages are excluded from compression by `getCompressibleMessages`
1559
+ // (they're outside the chunking range), so they won't appear in
1560
+ // any L1's sourceIds — no overlap with the recall pairs above.
1561
+ const allMessages = ctx.messageStore.getAll();
1562
+ const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1563
+ const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1564
+ for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1565
+ const m = allMessages[i];
1566
+ llmMessages.push({ participant: m.participant, content: m.content });
1567
+ }
1568
+
1569
+ // Any raw messages between the head and the chunk that aren't yet
1570
+ // represented by an L1 — usually empty in adaptive-resolution mode,
1571
+ // since chunking proceeds contiguously and L1s exist for everything
1572
+ // up to the chunk being processed.
1573
+ const chunkFirstId = chunk.messages[0]?.id;
1574
+ if (chunkFirstId) {
1575
+ const priorL1MessageIds = new Set<string>();
1576
+ for (const s of priorL1s) {
1577
+ for (const id of s.sourceIds) priorL1MessageIds.add(id);
1578
+ }
1579
+ const chunkStartIdx = allMessages.findIndex((m) => m.id === chunkFirstId);
1580
+ for (let i = headEndIdx; i < chunkStartIdx && i < allMessages.length; i++) {
1581
+ const m = allMessages[i];
1582
+ if (priorL1MessageIds.has(m.id)) continue;
1583
+ llmMessages.push({ participant: m.participant, content: m.content });
1584
+ }
1585
+ }
1586
+
1587
+ // ---- 3. In-band marker ----
1007
1588
  llmMessages.push({
1008
1589
  participant: 'Context Manager',
1009
- content: [{
1010
- type: 'text',
1011
- text: `[Context Manager] We are ready to form a long-term memory. Here is the conversation to remember:\n\n${chunkContent}\n\n${instruction}`,
1012
- }],
1590
+ content: [{ type: 'text', text: COMPRESSION_MARKER }],
1591
+ });
1592
+
1593
+ // ---- 4. Chunk messages raw ----
1594
+ for (const m of chunk.messages) {
1595
+ llmMessages.push({ participant: m.participant, content: m.content });
1596
+ }
1597
+
1598
+ // ---- 5. Instruction (reading-mode aware) ----
1599
+ //
1600
+ // When the chunk is a portion of a substantially larger sharded message
1601
+ // (≥ 2× chunk size), use the reading-mode instruction. It avoids the
1602
+ // "form a memory of what this contained" framing — which, for content
1603
+ // heavily first-person from someone other than the agent (a user-shared
1604
+ // doc), leads the model to adopt the content author's voice. Instead,
1605
+ // it asks what reading was like and what was learned, forcing the
1606
+ // model to reflect from its own vantage point in agent-first-person.
1607
+ const docContext = this.detectDocContext(chunk, ctx);
1608
+ const instructionText = docContext
1609
+ ? this.getReadingChunkInstruction(chunk, docContext.totalTokens, targetTokens)
1610
+ : this.getCompressionInstruction(chunk, targetTokens);
1611
+ llmMessages.push({
1612
+ participant: 'Context Manager',
1613
+ content: [{ type: 'text', text: instructionText }],
1013
1614
  });
1014
1615
 
1015
1616
  // Collapse consecutive same-participant messages for API compliance
1016
1617
  const collapsed = this.collapseConsecutiveMessages(llmMessages);
1017
1618
 
1619
+ // NO system prompt. The agent's identity is established by the head
1620
+ // (the actual conversation opening — user message + agent reply that
1621
+ // grounded the original instance). A system prompt would (a) add a
1622
+ // synthetic header the original instance never saw, disturbing KV
1623
+ // consistency between the summarizer and the original instance, and
1624
+ // (b) provide an alternative identity source that competes with the
1625
+ // structural one carried by the conversation itself. Anchoring
1626
+ // identity by the chronicle's actual head is more honest.
1018
1627
  const request: NormalizedRequest = {
1019
1628
  messages: collapsed.map(m => ({ participant: m.participant, content: m.content })),
1020
- system: 'You are forming autobiographical memories of a conversation.',
1021
1629
  config: {
1022
1630
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1023
1631
  maxTokens: Math.round(targetTokens * 1.5),
1024
1632
  },
1025
1633
  };
1026
1634
 
1635
+ const callStart = Date.now();
1636
+ let logResponse: string | undefined;
1637
+ let logError: string | undefined;
1638
+ let logSummaryId: string | undefined;
1639
+
1027
1640
  try {
1028
1641
  const response = await ctx.membrane.complete(request, { formatter: this.nativeFormatter });
1029
1642
  const summaryText = response.content
1030
1643
  .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
1031
1644
  .map(b => b.text)
1032
1645
  .join('\n');
1646
+ logResponse = summaryText;
1033
1647
 
1034
1648
  const messageIds = chunk.messages.map(m => m.id);
1035
1649
  const entry: SummaryEntry = {
@@ -1051,11 +1665,40 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1051
1665
  chunk.compressed = true;
1052
1666
  chunk.summaryId = entry.id;
1053
1667
  this._compressionCount++;
1668
+ logSummaryId = entry.id;
1054
1669
 
1055
1670
  this.checkMergeThreshold();
1056
1671
  } catch (error) {
1057
1672
  console.error('Failed to compress chunk (hierarchical):', error);
1673
+ logError = error instanceof Error ? error.message : String(error);
1058
1674
  throw error;
1675
+ } finally {
1676
+ logCompressionCall({
1677
+ operation: 'compress_l1',
1678
+ system: null,
1679
+ messages: collapsed.map((m) => ({
1680
+ participant: m.participant,
1681
+ // Flatten content for logging — store text only; binary content
1682
+ // would bloat the log and isn't typical in compression input.
1683
+ text: m.content
1684
+ .filter((b: ContentBlock) => b.type === 'text')
1685
+ .map((b: any) => b.text)
1686
+ .join(''),
1687
+ })),
1688
+ metadata: {
1689
+ chunk_message_ids: chunk.messages.map((m) => m.id),
1690
+ chunk_size: chunk.messages.length,
1691
+ prior_l1_count: priorL1s.length,
1692
+ has_doc_context: docContext !== null,
1693
+ doc_context: docContext,
1694
+ target_tokens: targetTokens,
1695
+ model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1696
+ latency_ms: Date.now() - callStart,
1697
+ summary_id: logSummaryId,
1698
+ },
1699
+ response: logResponse,
1700
+ error: logError,
1701
+ });
1059
1702
  }
1060
1703
  }
1061
1704
 
@@ -1069,6 +1712,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1069
1712
  * summaries when the queue eventually drains.
1070
1713
  */
1071
1714
  protected checkMergeThreshold(): void {
1715
+ if (this.config.speculativeProduction) {
1716
+ this.checkMergeThresholdRecursive();
1717
+ return;
1718
+ }
1719
+
1072
1720
  const threshold = this.config.mergeThreshold ?? 6;
1073
1721
 
1074
1722
  // IDs that are already part of a queued merge — exclude them from
@@ -1105,6 +1753,55 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1105
1753
  }
1106
1754
  }
1107
1755
 
1756
+ /**
1757
+ * Bottom-up speculative pre-producer (design doc §3.5 / §7.2).
1758
+ *
1759
+ * Recursive variant of `checkMergeThreshold` for the unbounded L_n
1760
+ * design. Walks every level present in the archive; for any level k
1761
+ * with ≥ N orphans (no parent), enqueues an L_{k+1} merge. After that
1762
+ * L_{k+1} is produced and `executeMerge` calls this again, the recursion
1763
+ * naturally cascades: 6 L1s → 1 L2; 6 L2s → 1 L3; 6 L3s → 1 L4; ...
1764
+ *
1765
+ * Only fires when `config.speculativeProduction` is true. Default true
1766
+ * for adaptiveResolution=true, false otherwise. The non-speculative path
1767
+ * (above) preserves the original L1→L2→L3 behavior for non-adaptive
1768
+ * deployments.
1769
+ */
1770
+ protected checkMergeThresholdRecursive(): void {
1771
+ const threshold = this.config.mergeThreshold ?? 6;
1772
+
1773
+ // Build per-level sets of source-ids already enqueued for merging,
1774
+ // so we don't re-enqueue them while a merge is pending.
1775
+ const queuedSources = new Map<number, Set<string>>();
1776
+ for (const m of this.mergeQueue) {
1777
+ // m.level is the TARGET level; the sources are at level (m.level - 1).
1778
+ const sourceLevel = m.level - 1;
1779
+ if (!queuedSources.has(sourceLevel)) queuedSources.set(sourceLevel, new Set());
1780
+ for (const id of m.sourceIds) queuedSources.get(sourceLevel)!.add(id);
1781
+ }
1782
+
1783
+ // Walk every level present in the archive. Iterate from low to high
1784
+ // so when an L_{k+1} merge is enqueued and immediately produced, this
1785
+ // same check sees the new L_{k+1} on the next call and can roll up.
1786
+ let maxLevel = 0;
1787
+ for (const s of this.summaries) {
1788
+ if (s.level > maxLevel) maxLevel = s.level;
1789
+ }
1790
+ for (let level = 1; level <= maxLevel; level++) {
1791
+ const queued = queuedSources.get(level) ?? new Set();
1792
+ const unmerged = this.summaries.filter(
1793
+ s => s.level === level && !getSummaryParentId(s) && !queued.has(s.id),
1794
+ );
1795
+ if (unmerged.length >= threshold) {
1796
+ const toMerge = unmerged.slice(0, threshold);
1797
+ this.enqueueMerge({
1798
+ level: level + 1,
1799
+ sourceIds: toMerge.map(s => s.id),
1800
+ });
1801
+ }
1802
+ }
1803
+ }
1804
+
1108
1805
  /**
1109
1806
  * Merge N summaries at one level into a single summary at the next level.
1110
1807
  * Uses self-voice consolidation prompt.
@@ -1141,57 +1838,225 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1141
1838
  const targetTokens = this.config.summaryTargetTokens ?? 2000;
1142
1839
  const participant = this.config.summaryParticipant ?? 'Claude';
1143
1840
 
1144
- // Build message array
1841
+ // Build the merge prompt with one-level-deeper target expansion +
1842
+ // prefix of older context:
1843
+ //
1844
+ // 1. PREFIX — head messages + prior L1 recall pairs for content
1845
+ // that comes chronologically BEFORE the merge range. "Fill
1846
+ // lower orbitals first" per the spec: regardless of how
1847
+ // compressed the live view is, the summarizer always gets L1
1848
+ // fidelity for prior content. Older L2/L3 markers exist for
1849
+ // live-view compactness, not for the summarizer.
1850
+ //
1851
+ // 2. TARGET — the sources expanded ONE LEVEL DEEPER than they
1852
+ // themselves are. For L2 merge (sources at L1): expand to
1853
+ // raw L0 messages — the model sees the actual conversation
1854
+ // that the 6 L1s consolidate. For L3 merge (sources at L2):
1855
+ // expand to the L1s under each L2 (36 L1s as recall pairs).
1856
+ // For L_n merge (sources at L_{n-1}): expand to L_{n-2}.
1857
+ // This gives the model substantively more content to ground
1858
+ // the consolidation in than just the 6 surface summaries.
1859
+ //
1860
+ // 3. INSTRUCTION — "consolidate N memories preserving the
1861
+ // through-line, in first person".
1862
+ //
1863
+ // No tail-after-merge: same as-of principle as L1 compression. The
1864
+ // consolidation is being formed at the moment the last source was
1865
+ // ready, so nothing after that is visible.
1145
1866
  const llmMessages: Array<{ participant: string; content: ContentBlock[] }> = [];
1146
1867
 
1147
- // Higher-level context (anti-redundant)
1148
- if (targetLevel === 2) {
1149
- // For L2 merge: show L3 summaries as context
1150
- const shownL3 = this.summaries.filter(s => s.level === 3 && !s.mergedInto);
1151
- for (const s of shownL3) {
1152
- llmMessages.push({
1153
- participant,
1154
- content: [{ type: 'text', text: s.content }],
1155
- });
1868
+ // Build lookup maps
1869
+ const summariesById = new Map<string, SummaryEntry>();
1870
+ for (const s of this.summaries) summariesById.set(s.id, s);
1871
+ const allMessages = ctx.messageStore.getAll();
1872
+ const messageById = new Map<MessageId, typeof allMessages[number]>();
1873
+ for (const m of allMessages) messageById.set(m.id, m);
1874
+
1875
+ // Compute every leaf message id covered by this merge's lineage —
1876
+ // these are part of the TARGET and must not also appear in the
1877
+ // PREFIX as head content.
1878
+ const sourceLeafIds = new Set<MessageId>();
1879
+ const collectLeaves = (s: SummaryEntry): void => {
1880
+ if (s.sourceLevel === 0) {
1881
+ for (const id of s.sourceIds) sourceLeafIds.add(id);
1882
+ } else {
1883
+ for (const childId of s.sourceIds) {
1884
+ const child = summariesById.get(childId);
1885
+ if (child) collectLeaves(child);
1886
+ }
1156
1887
  }
1888
+ };
1889
+ for (const src of sources) collectLeaves(src);
1890
+
1891
+ // Find the start of the merge range in the message store.
1892
+ const mergeFirstMsgId = sources[0].sourceRange.first;
1893
+ const mergeStartIdx = allMessages.findIndex((m) => m.id === mergeFirstMsgId);
1894
+
1895
+ // ---- 1a. HEAD WINDOW (raw, ALWAYS present) ----
1896
+ //
1897
+ // The head window is the foundational identity anchor — the actual
1898
+ // opening of the chronicle. It establishes who is speaking to whom.
1899
+ // Without it, when the merge target's content is heavily first-person
1900
+ // from someone other than the agent, the agent loses its first-person
1901
+ // grounding and drifts into the content author's voice.
1902
+ const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1903
+ const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1904
+ for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1905
+ const m = allMessages[i];
1906
+ llmMessages.push({ participant: m.participant, content: m.content });
1907
+ }
1908
+
1909
+ // ---- 1b. PRIOR L1 RECALL PAIRS (chronologically before merge range) ----
1910
+ // L1s whose entire source range is before the merge range and that
1911
+ // aren't part of the merge tree. Sort by source position.
1912
+ const priorL1s = this.summaries
1913
+ .filter((s) => s.level === 1)
1914
+ .filter((s) => {
1915
+ // Exclude any L1 that's an ancestor of our merge target
1916
+ for (const lid of s.sourceIds) if (sourceLeafIds.has(lid)) return false;
1917
+ // Include only if it starts strictly before the merge range
1918
+ const firstIdx = allMessages.findIndex((m) => m.id === s.sourceRange.first);
1919
+ return firstIdx >= 0 && (mergeStartIdx < 0 || firstIdx < mergeStartIdx);
1920
+ })
1921
+ .sort((a, b) => {
1922
+ const ai = allMessages.findIndex((m) => m.id === a.sourceRange.first);
1923
+ const bi = allMessages.findIndex((m) => m.id === b.sourceRange.first);
1924
+ return ai - bi;
1925
+ });
1926
+
1927
+ const priorL1MessageIds = new Set<MessageId>();
1928
+ for (const s of priorL1s) {
1929
+ for (const id of s.sourceIds) priorL1MessageIds.add(id);
1157
1930
  }
1158
- // For L3 merge: no higher context exists
1159
1931
 
1160
- // The source summaries as agent's own memories
1161
- for (const source of sources) {
1932
+ for (const s of priorL1s) {
1933
+ llmMessages.push({
1934
+ participant: 'Context Manager',
1935
+ content: [{ type: 'text', text: `[CM] Recall memory ${s.id}.` }],
1936
+ });
1162
1937
  llmMessages.push({
1163
1938
  participant,
1164
- content: [{ type: 'text', text: source.content }],
1939
+ content: [{ type: 'text', text: s.content }],
1165
1940
  });
1166
1941
  }
1167
1942
 
1168
- // Consolidation instruction
1169
- const mergeInstruction = this.getMergeInstruction(targetLevel, sources, targetTokens);
1943
+ // Raw middle: any messages between the head window and the merge
1944
+ // range that aren't covered by a prior L1 or the merge tree.
1945
+ // Usually empty (chunking is contiguous).
1946
+ if (mergeStartIdx >= 0) {
1947
+ for (let i = headEndIdx; i < mergeStartIdx; i++) {
1948
+ const m = allMessages[i];
1949
+ if (priorL1MessageIds.has(m.id)) continue;
1950
+ if (sourceLeafIds.has(m.id)) continue;
1951
+ llmMessages.push({ participant: m.participant, content: m.content });
1952
+ }
1953
+ }
1954
+
1955
+ // ---- 2. TARGET: expand sources one level deeper ----
1956
+ // For L2 (sources at L1, sourceLevel=0): expand to raw L0 messages.
1957
+ // For L3+ (sources at L_{n-1}, sourceLevel=n-2): expand to L_{n-2}
1958
+ // summaries as recall pairs.
1959
+ for (const src of sources) {
1960
+ if (src.sourceLevel === 0) {
1961
+ // Source is an L1; its sourceIds are raw message ids. Emit them raw.
1962
+ for (const messageId of src.sourceIds) {
1963
+ const m = messageById.get(messageId);
1964
+ if (m) {
1965
+ llmMessages.push({ participant: m.participant, content: m.content });
1966
+ }
1967
+ }
1968
+ } else {
1969
+ // Source is L2+; its sourceIds point to summaries one level
1970
+ // below. Emit each as a recall pair.
1971
+ for (const childId of src.sourceIds) {
1972
+ const child = summariesById.get(childId);
1973
+ if (!child) continue;
1974
+ llmMessages.push({
1975
+ participant: 'Context Manager',
1976
+ content: [{ type: 'text', text: `[CM] Recall memory ${child.id}.` }],
1977
+ });
1978
+ llmMessages.push({
1979
+ participant,
1980
+ content: [{ type: 'text', text: child.content }],
1981
+ });
1982
+ }
1983
+ }
1984
+ }
1985
+
1986
+ // ---- 3. INSTRUCTION ----
1987
+ // sourceLevelShown is the level of content the model actually sees
1988
+ // (one level below the sources themselves).
1989
+ const sourceLevelShown =
1990
+ sources[0].sourceLevel === 0 ? 0 : sources[0].level - 1;
1991
+
1992
+ // Reading-mode detection: when ALL the merge's leaf messages are
1993
+ // shards of the same bodyGroup, we know the agent was reading a
1994
+ // substantially larger message rather than conversing. The
1995
+ // reading-mode merge instruction asks what reading the stretch was
1996
+ // like instead of asking for an impersonal consolidation, which
1997
+ // forces the agent's vantage point and prevents drift into the
1998
+ // content author's voice. Same principle as the L1 case.
1999
+ let mergeReadingContext: { totalTokens: number } | null = null;
2000
+ if (sourceLeafIds.size > 0) {
2001
+ const leafBodyGroupIds = new Set<string | undefined>();
2002
+ for (const leafId of sourceLeafIds) {
2003
+ const m = messageById.get(leafId);
2004
+ leafBodyGroupIds.add(m?.bodyGroupId);
2005
+ }
2006
+ if (leafBodyGroupIds.size === 1) {
2007
+ const groupId = [...leafBodyGroupIds][0];
2008
+ if (groupId) {
2009
+ let totalTokens = 0;
2010
+ for (const m of allMessages) {
2011
+ if (m.bodyGroupId === groupId) {
2012
+ totalTokens += ctx.messageStore.estimateTokens(m);
2013
+ }
2014
+ }
2015
+ mergeReadingContext = { totalTokens };
2016
+ }
2017
+ }
2018
+ }
2019
+ const mergeInstructionText = mergeReadingContext
2020
+ ? this.getReadingMergeInstruction(
2021
+ targetLevel,
2022
+ sources,
2023
+ mergeReadingContext.totalTokens,
2024
+ targetTokens,
2025
+ )
2026
+ : this.getMergeInstruction(targetLevel, sources, targetTokens);
1170
2027
  llmMessages.push({
1171
2028
  participant: 'Context Manager',
1172
2029
  content: [{
1173
2030
  type: 'text',
1174
- text: `[Context Manager] ${mergeInstruction}`,
2031
+ text: mergeInstructionText,
1175
2032
  }],
1176
2033
  });
1177
2034
 
1178
2035
  const collapsed = this.collapseConsecutiveMessages(llmMessages);
1179
2036
 
2037
+ // NO system prompt — identity is established by the head window
2038
+ // (present at the start of llmMessages above) and by the prior
2039
+ // recall pairs. Same rationale as compressChunkHierarchical.
1180
2040
  const request: NormalizedRequest = {
1181
2041
  messages: collapsed.map(m => ({ participant: m.participant, content: m.content })),
1182
- system: 'You are forming autobiographical memories of a conversation.',
1183
2042
  config: {
1184
2043
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1185
2044
  maxTokens: Math.round(targetTokens * 1.5),
1186
2045
  },
1187
2046
  };
1188
2047
 
2048
+ const callStart = Date.now();
2049
+ let logResponse: string | undefined;
2050
+ let logError: string | undefined;
2051
+ let logNewSummaryId: string | undefined;
2052
+
1189
2053
  try {
1190
2054
  const response = await ctx.membrane.complete(request, { formatter: this.nativeFormatter });
1191
2055
  const mergedText = response.content
1192
2056
  .filter((b): b is { type: 'text'; text: string } => b.type === 'text')
1193
2057
  .map(b => b.text)
1194
2058
  .join('\n');
2059
+ logResponse = mergedText;
1195
2060
 
1196
2061
  // Compute source range from constituent summaries
1197
2062
  const sourceRange = {
@@ -1210,6 +2075,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1210
2075
  sourceRange,
1211
2076
  created: Date.now(),
1212
2077
  };
2078
+ logNewSummaryId = newEntry.id;
1213
2079
 
1214
2080
  // Append the new merged entry first, then mark sources. Persist each
1215
2081
  // mergedInto edit individually so chronicle reflects the same shape as
@@ -1226,10 +2092,602 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1226
2092
  this.checkMergeThreshold();
1227
2093
  } catch (error) {
1228
2094
  console.error(`Failed to merge summaries into L${targetLevel}:`, error);
2095
+ logError = error instanceof Error ? error.message : String(error);
1229
2096
  throw error;
2097
+ } finally {
2098
+ logCompressionCall({
2099
+ operation: `merge_l${targetLevel}`,
2100
+ system: null,
2101
+ messages: collapsed.map((m) => ({
2102
+ participant: m.participant,
2103
+ text: m.content
2104
+ .filter((b: ContentBlock) => b.type === 'text')
2105
+ .map((b: any) => b.text)
2106
+ .join(''),
2107
+ })),
2108
+ metadata: {
2109
+ target_level: targetLevel,
2110
+ source_ids: sourceIds,
2111
+ source_level: sources[0]?.level ?? null,
2112
+ source_level_shown: sourceLevelShown,
2113
+ target_tokens: targetTokens,
2114
+ model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
2115
+ latency_ms: Date.now() - callStart,
2116
+ summary_id: logNewSummaryId,
2117
+ },
2118
+ response: logResponse,
2119
+ error: logError,
2120
+ });
1230
2121
  }
1231
2122
  }
1232
2123
 
2124
+ // ============================================================================
2125
+ // Adaptive resolution (picker-driven) path
2126
+ // ============================================================================
2127
+
2128
+ /**
2129
+ * Select context entries using the adaptive-resolution picker.
2130
+ *
2131
+ * Builds per-message PickerChunks from compressible messages, runs the
2132
+ * configured FoldingStrategy under token-budget pressure, and emits the
2133
+ * resulting per-message resolutions as ContextEntry[]. Adjacent messages
2134
+ * sharing the same L_k ancestor emit the recall pair once.
2135
+ *
2136
+ * See `docs/adaptive-resolution-design.md` §3, §5.
2137
+ */
2138
+ protected selectAdaptive(store: MessageStoreView, budget: TokenBudget): ContextEntry[] {
2139
+ const entries: ContextEntry[] = [];
2140
+ const maxTokens = budget.maxTokens - budget.reserveForResponse;
2141
+ const messages = store.getAll();
2142
+ const msgCap = this.config.maxMessageTokens;
2143
+
2144
+ // ----- 1. Build head/tail sets and emit head entries -----
2145
+ const headStart = this.getHeadWindowStartIndex(store);
2146
+ const headEnd = this.getHeadWindowEnd(store);
2147
+ const recentStart = this.getRecentWindowStart(store);
2148
+
2149
+ const headMessageIds = new Set<MessageId>();
2150
+ const tailMessageIds = new Set<MessageId>();
2151
+ let headTokens = 0;
2152
+ let tailTokens = 0;
2153
+
2154
+ let totalTokens = 0;
2155
+
2156
+ // Emit head entries verbatim
2157
+ for (let i = headStart; i < headEnd && i < messages.length; i++) {
2158
+ const msg = messages[i];
2159
+ const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2160
+ const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2161
+ if (totalTokens + tokens > maxTokens) break;
2162
+ entries.push({
2163
+ index: entries.length,
2164
+ sourceMessageId: msg.id,
2165
+ sourceRelation: 'copy',
2166
+ participant: msg.participant,
2167
+ content,
2168
+ });
2169
+ totalTokens += tokens;
2170
+ headMessageIds.add(msg.id);
2171
+ headTokens += tokens;
2172
+ }
2173
+ if (entries.length > 0) {
2174
+ entries[entries.length - 1].cacheMarker = true;
2175
+ }
2176
+
2177
+ // Compute tail message IDs (will be emitted at end)
2178
+ const effectiveRecentStart = Math.max(recentStart, headEnd);
2179
+ for (let i = effectiveRecentStart; i < messages.length; i++) {
2180
+ const msg = messages[i];
2181
+ const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2182
+ tailMessageIds.add(msg.id);
2183
+ tailTokens += tokens;
2184
+ }
2185
+
2186
+ // ----- 2. Build PickerChunks for messages in the middle -----
2187
+ // For each compressible (non-head, non-tail) message we create one
2188
+ // PickerChunk. Its l1Id is determined by the existing chunks that
2189
+ // group messages into L1 summaries.
2190
+ const chunksByMessageId = new Map<MessageId, Chunk>();
2191
+ for (const ch of this.chunks) {
2192
+ for (const m of ch.messages) {
2193
+ chunksByMessageId.set(m.id, ch);
2194
+ }
2195
+ }
2196
+
2197
+ // Pinned-position set so the picker doesn't fold messages the user
2198
+ // explicitly marked as keep-raw. Built once and reused.
2199
+ const pinnedSet = this.pinnedPositions(messages);
2200
+
2201
+ // O(1) summary lookup for findAncestorAt — avoids O(summaries) find()
2202
+ // calls during emission.
2203
+ const summariesById = new Map<string, SummaryEntry>();
2204
+ for (const s of this.summaries) summariesById.set(s.id, s);
2205
+
2206
+ const pickerChunks: PickerChunk[] = [];
2207
+ for (let i = headEnd; i < effectiveRecentStart && i < messages.length; i++) {
2208
+ const msg = messages[i];
2209
+ const ch = chunksByMessageId.get(msg.id);
2210
+ const tokens = msgCap > 0
2211
+ ? Math.min(store.estimateTokens(msg), msgCap + 50)
2212
+ : store.estimateTokens(msg);
2213
+ pickerChunks.push({
2214
+ id: msg.id,
2215
+ sequence: i,
2216
+ rawTokens: tokens,
2217
+ currentResolution: this.resolutions.get(msg.id) ?? 0,
2218
+ lockedByAgent: this.locked.has(msg.id),
2219
+ pinned: pinnedSet.has(i),
2220
+ l1Id: ch?.summaryId,
2221
+ });
2222
+ }
2223
+
2224
+ // Also include head and tail in PickerChunks (so token accounting matches)
2225
+ // — but mark them as in-head/in-tail so the picker won't fold them.
2226
+ for (let i = headStart; i < headEnd && i < messages.length; i++) {
2227
+ const msg = messages[i];
2228
+ const tokens = msgCap > 0
2229
+ ? Math.min(store.estimateTokens(msg), msgCap + 50)
2230
+ : store.estimateTokens(msg);
2231
+ pickerChunks.push({
2232
+ id: msg.id,
2233
+ sequence: i,
2234
+ rawTokens: tokens,
2235
+ currentResolution: 0,
2236
+ lockedByAgent: this.locked.has(msg.id),
2237
+ pinned: true, // treat head as pinned for picker purposes
2238
+ l1Id: undefined,
2239
+ });
2240
+ }
2241
+ for (let i = effectiveRecentStart; i < messages.length; i++) {
2242
+ const msg = messages[i];
2243
+ const tokens = msgCap > 0
2244
+ ? Math.min(store.estimateTokens(msg), msgCap + 50)
2245
+ : store.estimateTokens(msg);
2246
+ pickerChunks.push({
2247
+ id: msg.id,
2248
+ sequence: i,
2249
+ rawTokens: tokens,
2250
+ currentResolution: 0,
2251
+ lockedByAgent: this.locked.has(msg.id),
2252
+ pinned: true, // treat tail as pinned for picker purposes
2253
+ l1Id: undefined,
2254
+ });
2255
+ }
2256
+
2257
+ // ----- 3. Build summaries map and recall-pair tokens -----
2258
+ const summariesMap = new Map<SummaryId, SummaryEntry>();
2259
+ const recallPairTokens = new Map<SummaryId, number>();
2260
+ for (const s of this.summaries) {
2261
+ summariesMap.set(s.id, s);
2262
+ // recall pair = the summary's text wrapped as a Q&A pair. Approximate
2263
+ // as s.tokens + small overhead for the "What do you remember?" label.
2264
+ recallPairTokens.set(s.id, s.tokens + 20);
2265
+ }
2266
+
2267
+ // ----- 4. Run the picker -----
2268
+ const totalBudget = maxTokens - totalTokens; // tokens left after head
2269
+ const slack = this.config.compressionSlackRatio ?? 0.1;
2270
+ const foldingBudget: FoldingBudget = {
2271
+ totalBudget,
2272
+ targetBudget: totalBudget * (1 - slack),
2273
+ slack,
2274
+ };
2275
+
2276
+ const headSetForPicker = new Set<ChunkId>(headMessageIds);
2277
+ const tailSetForPicker = new Set<ChunkId>(tailMessageIds);
2278
+
2279
+ const picker = this.getAdaptivePicker();
2280
+ const result = picker.run(
2281
+ {
2282
+ chunks: pickerChunks,
2283
+ summaries: summariesMap,
2284
+ recallPairTokens,
2285
+ headChunkIds: headSetForPicker,
2286
+ tailChunkIds: tailSetForPicker,
2287
+ headTokens,
2288
+ tailTokens,
2289
+ },
2290
+ foldingBudget
2291
+ );
2292
+
2293
+ // Commit the new resolutions back to strategy state for next compile.
2294
+ // Persist to chronicle only if anything actually changed — avoids
2295
+ // unnecessary state-slot writes on no-op compiles (which is the common
2296
+ // case in steady state with slack).
2297
+ let resolutionsChanged = false;
2298
+ let deepestLevel = 0;
2299
+ for (const [id, level] of result.finalResolutions) {
2300
+ if (headMessageIds.has(id) || tailMessageIds.has(id)) continue;
2301
+ if (this.locked.has(id)) continue;
2302
+ const prev = this.resolutions.get(id) ?? 0;
2303
+ if (prev !== level) {
2304
+ this.resolutions.set(id, level);
2305
+ resolutionsChanged = true;
2306
+ }
2307
+ if (level > deepestLevel) deepestLevel = level;
2308
+ }
2309
+ if (resolutionsChanged) {
2310
+ this.persistResolutions();
2311
+ }
2312
+
2313
+ // Wire produce ops into the strategy's own production queues so that
2314
+ // requested-but-not-yet-existing summaries actually get built. The
2315
+ // speculative pre-producer covers most cases ambiently, but when it is
2316
+ // disabled (`speculativeProduction: false`) or hasn't reached the level
2317
+ // the picker just asked for, the request would otherwise be dropped and
2318
+ // the picker would re-emit it on every subsequent compile. Handling it
2319
+ // here makes the produce path observable and convergent.
2320
+ //
2321
+ // The actual compression/merge work runs asynchronously via the next
2322
+ // `tick()` invocation (or the speculative drain kicked from
2323
+ // `onNewMessage`). This call only enqueues; it does not await.
2324
+ if (result.produced.length > 0) {
2325
+ this.handleProducedOps(result.produced);
2326
+ }
2327
+
2328
+ // Hard-fail check: if the picker exhausted itself but the final render
2329
+ // would still exceed the HARD budget (not just the soft target), surface
2330
+ // an OverBudgetError to the host rather than silently dropping entries.
2331
+ // The strategy has done all it can; the application has to decide what
2332
+ // to do next (raise budget, switch model, drop windows, etc.).
2333
+ if (result.exhausted && result.finalTokens > totalBudget) {
2334
+ throw new OverBudgetError({
2335
+ budget: totalBudget,
2336
+ actual: result.finalTokens,
2337
+ diagnostics: {
2338
+ headTokens,
2339
+ tailTokens,
2340
+ middleTokens: Math.max(0, result.finalTokens - headTokens - tailTokens),
2341
+ middleChunkCount: pickerChunks.length - headMessageIds.size - tailMessageIds.size,
2342
+ deepestLevel,
2343
+ },
2344
+ });
2345
+ }
2346
+
2347
+ // ----- 5. Emit middle entries in source order -----
2348
+ // Walk middle messages. Handle two cases:
2349
+ // - bodyGroupId set: collect all consecutive shards from the same group,
2350
+ // emit ONE combined entry with concatenated content (raw shards + inline
2351
+ // summary text for folded shards). This preserves KV — the model sees
2352
+ // one continuous user message instead of N turns.
2353
+ // - bodyGroupId absent: emit normally (raw L0 message, or Q+A summary pair).
2354
+ const emittedAncestors = new Set<SummaryId>();
2355
+ const summaryLabel = this.config.summaryContextLabel ?? 'What do you remember from earlier?';
2356
+ const summaryParticipant = this.config.summaryParticipant ?? 'Claude';
2357
+
2358
+ let i = headEnd;
2359
+ while (i < effectiveRecentStart && i < messages.length) {
2360
+ const msg = messages[i];
2361
+
2362
+ if (msg.bodyGroupId) {
2363
+ // Collect the full run of consecutive shards sharing this bodyGroupId.
2364
+ const groupId = msg.bodyGroupId;
2365
+ const groupStart = i;
2366
+ while (
2367
+ i < effectiveRecentStart &&
2368
+ i < messages.length &&
2369
+ messages[i].bodyGroupId === groupId
2370
+ ) {
2371
+ i++;
2372
+ }
2373
+ const groupMessages = messages.slice(groupStart, i);
2374
+ // Sort by shardIndex to ensure byte-faithful ordering.
2375
+ const sortedShards = [...groupMessages].sort(
2376
+ (a, b) => (a.shardIndex ?? 0) - (b.shardIndex ?? 0)
2377
+ );
2378
+
2379
+ // Walk shards in order, accumulating "runs":
2380
+ // - a 'raw' run is consecutive shards at L0; flushed as ONE User
2381
+ // message with their text concatenated.
2382
+ // - a 'summary' run is consecutive shards under the same L_k
2383
+ // ancestor; flushed as a Q+A recall pair (Context Manager
2384
+ // question + summaryParticipant answer), emitted once per
2385
+ // distinct ancestor.
2386
+ // The run breaks (and the previous run flushes) when:
2387
+ // - resolution transitions raw ↔ folded
2388
+ // - the L_k ancestor changes
2389
+ type Run =
2390
+ | { kind: 'raw'; parts: string[] }
2391
+ | { kind: 'summary'; ancestor: SummaryEntry };
2392
+ let currentRun: Run | null = null;
2393
+ const participant = sortedShards[0].participant;
2394
+
2395
+ const flushRun = (): boolean => {
2396
+ if (!currentRun) return true;
2397
+ if (currentRun.kind === 'raw') {
2398
+ const text = currentRun.parts.join('');
2399
+ const content: ContentBlock[] = [{ type: 'text', text }];
2400
+ // Deliberately do NOT apply maxMessageTokens here: the picker
2401
+ // is the authority on how much of the doc renders raw vs.
2402
+ // summarized. Truncating the composite would silently lose
2403
+ // doc content that the picker explicitly chose to keep raw.
2404
+ // (`maxMessageTokens` is for per-message caps on chat / tool
2405
+ // results, not for sharded bodyGroup composites.)
2406
+ const tokens = this.estimateTokens(content);
2407
+ if (totalTokens + tokens > maxTokens) {
2408
+ currentRun = null;
2409
+ return false;
2410
+ }
2411
+ entries.push({
2412
+ index: entries.length,
2413
+ sourceMessageId: undefined,
2414
+ sourceRelation: 'copy',
2415
+ participant,
2416
+ content,
2417
+ });
2418
+ totalTokens += tokens;
2419
+ } else {
2420
+ // summary run — emit Q+A pair, dedup at the strategy level
2421
+ const ancestor = currentRun.ancestor;
2422
+ if (!emittedAncestors.has(ancestor.id)) {
2423
+ emittedAncestors.add(ancestor.id);
2424
+ const questionEntry: ContextEntry = {
2425
+ index: entries.length,
2426
+ participant: 'Context Manager',
2427
+ content: [{ type: 'text', text: summaryLabel }],
2428
+ sourceRelation: 'derived',
2429
+ };
2430
+ const answerContent: ContentBlock[] = [{ type: 'text', text: ancestor.content }];
2431
+ const answerEntry: ContextEntry = {
2432
+ index: entries.length + 1,
2433
+ participant: summaryParticipant,
2434
+ content: msgCap > 0 ? this.truncateContent(answerContent, msgCap) : answerContent,
2435
+ sourceRelation: 'derived',
2436
+ };
2437
+ const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2438
+ if (totalTokens + pairTokens > maxTokens) {
2439
+ currentRun = null;
2440
+ return false;
2441
+ }
2442
+ entries.push(questionEntry);
2443
+ entries.push(answerEntry);
2444
+ totalTokens += pairTokens;
2445
+ }
2446
+ }
2447
+ currentRun = null;
2448
+ return true;
2449
+ };
2450
+
2451
+ let budgetExhausted = false;
2452
+ for (const shard of sortedShards) {
2453
+ const resolution = result.finalResolutions.get(shard.id) ?? 0;
2454
+ if (resolution === 0) {
2455
+ if (currentRun?.kind !== 'raw') {
2456
+ if (!flushRun()) { budgetExhausted = true; break; }
2457
+ currentRun = { kind: 'raw', parts: [] };
2458
+ }
2459
+ for (const block of shard.content) {
2460
+ if (block.type === 'text') (currentRun as { kind: 'raw'; parts: string[] }).parts.push(block.text);
2461
+ }
2462
+ } else {
2463
+ const ancestor = this.findAncestorAt(shard.id, resolution, chunksByMessageId, summariesById);
2464
+ if (!ancestor) {
2465
+ // Fall back to raw
2466
+ if (currentRun?.kind !== 'raw') {
2467
+ if (!flushRun()) { budgetExhausted = true; break; }
2468
+ currentRun = { kind: 'raw', parts: [] };
2469
+ }
2470
+ for (const block of shard.content) {
2471
+ if (block.type === 'text') (currentRun as { kind: 'raw'; parts: string[] }).parts.push(block.text);
2472
+ }
2473
+ continue;
2474
+ }
2475
+ // If we're already in a summary run for the SAME ancestor, this
2476
+ // shard is covered — skip silently.
2477
+ if (currentRun?.kind === 'summary' && currentRun.ancestor.id === ancestor.id) {
2478
+ continue;
2479
+ }
2480
+ if (!flushRun()) { budgetExhausted = true; break; }
2481
+ currentRun = { kind: 'summary', ancestor };
2482
+ }
2483
+ }
2484
+ if (!budgetExhausted) {
2485
+ if (!flushRun()) budgetExhausted = true;
2486
+ }
2487
+ if (budgetExhausted) break;
2488
+ continue;
2489
+ }
2490
+
2491
+ // Non-shard path: existing behavior.
2492
+ const resolution = result.finalResolutions.get(msg.id) ?? 0;
2493
+ if (resolution === 0) {
2494
+ const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2495
+ const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2496
+ if (totalTokens + tokens > maxTokens) break;
2497
+ entries.push({
2498
+ index: entries.length,
2499
+ sourceMessageId: msg.id,
2500
+ sourceRelation: 'copy',
2501
+ participant: msg.participant,
2502
+ content,
2503
+ });
2504
+ totalTokens += tokens;
2505
+ i++;
2506
+ } else {
2507
+ const ancestor = this.findAncestorAt(msg.id, resolution, chunksByMessageId, summariesById);
2508
+ if (!ancestor) {
2509
+ const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2510
+ const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2511
+ if (totalTokens + tokens > maxTokens) break;
2512
+ entries.push({
2513
+ index: entries.length,
2514
+ sourceMessageId: msg.id,
2515
+ sourceRelation: 'copy',
2516
+ participant: msg.participant,
2517
+ content,
2518
+ });
2519
+ totalTokens += tokens;
2520
+ i++;
2521
+ continue;
2522
+ }
2523
+ if (emittedAncestors.has(ancestor.id)) {
2524
+ i++;
2525
+ continue;
2526
+ }
2527
+ emittedAncestors.add(ancestor.id);
2528
+ const questionEntry: ContextEntry = {
2529
+ index: entries.length,
2530
+ participant: 'Context Manager',
2531
+ content: [{ type: 'text', text: summaryLabel }],
2532
+ sourceRelation: 'derived',
2533
+ };
2534
+ const answerContent: ContentBlock[] = [{ type: 'text', text: ancestor.content }];
2535
+ const answerEntry: ContextEntry = {
2536
+ index: entries.length + 1,
2537
+ participant: summaryParticipant,
2538
+ content: msgCap > 0 ? this.truncateContent(answerContent, msgCap) : answerContent,
2539
+ sourceRelation: 'derived',
2540
+ };
2541
+ const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2542
+ if (totalTokens + pairTokens > maxTokens) break;
2543
+ entries.push(questionEntry);
2544
+ entries.push(answerEntry);
2545
+ totalTokens += pairTokens;
2546
+ i++;
2547
+ }
2548
+ }
2549
+
2550
+ // ----- 6. Emit tail entries newest-first eviction (matches existing behavior) -----
2551
+ this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
2552
+
2553
+ // ----- 7. Post-process: merge consecutive raw entries from the same bodyGroup -----
2554
+ // Both head and tail emission paths emit shards as separate ContextEntries.
2555
+ // The middle path already merges consecutive same-bodyGroup raw shards into
2556
+ // one composite entry, but head/tail don't. This pass closes that gap so
2557
+ // a sharded message renders as ONE API message regardless of which region
2558
+ // it falls into (preserves KV cache through region transitions).
2559
+ const merged = this.mergeAdjacentBodyGroupRaw(entries, store);
2560
+
2561
+ this.trimOrphanedToolUse(merged);
2562
+ return merged;
2563
+ }
2564
+
2565
+ /**
2566
+ * Walk an entries array; for every run of consecutive entries that
2567
+ * (a) have sourceRelation: 'copy' (raw, not a synthesized recall pair)
2568
+ * (b) have sourceMessageId pointing to messages in the same bodyGroup
2569
+ * merge them into one composite entry whose body is the byte-faithful
2570
+ * concatenation of their text content. Other entries pass through.
2571
+ *
2572
+ * Reindexes the returned array.
2573
+ */
2574
+ protected mergeAdjacentBodyGroupRaw(
2575
+ entries: ContextEntry[],
2576
+ store: MessageStoreView
2577
+ ): ContextEntry[] {
2578
+ if (entries.length === 0) return entries;
2579
+
2580
+ // Look up bodyGroupId by sourceMessageId via the message store.
2581
+ const groupOf = (sourceMessageId?: string): string | undefined => {
2582
+ if (!sourceMessageId) return undefined;
2583
+ const m = store.get(sourceMessageId);
2584
+ return m?.bodyGroupId;
2585
+ };
2586
+
2587
+ const out: ContextEntry[] = [];
2588
+ let i = 0;
2589
+ while (i < entries.length) {
2590
+ const entry = entries[i];
2591
+ const groupId = entry.sourceRelation === 'copy' ? groupOf(entry.sourceMessageId) : undefined;
2592
+ if (!groupId) {
2593
+ out.push({ ...entry, index: out.length });
2594
+ i++;
2595
+ continue;
2596
+ }
2597
+ // Collect run of consecutive raw entries with same bodyGroupId.
2598
+ const run: ContextEntry[] = [entry];
2599
+ let j = i + 1;
2600
+ while (
2601
+ j < entries.length &&
2602
+ entries[j].sourceRelation === 'copy' &&
2603
+ groupOf(entries[j].sourceMessageId) === groupId
2604
+ ) {
2605
+ run.push(entries[j]);
2606
+ j++;
2607
+ }
2608
+ if (run.length === 1) {
2609
+ out.push({ ...entry, index: out.length });
2610
+ i++;
2611
+ continue;
2612
+ }
2613
+ // Sort the run by the underlying shardIndex to ensure byte-faithful
2614
+ // ordering. (Head/tail emission keeps chronological order, but defending
2615
+ // against reorderings is cheap.)
2616
+ const sortedRun = [...run].sort((a, b) => {
2617
+ const ma = a.sourceMessageId ? store.get(a.sourceMessageId) : null;
2618
+ const mb = b.sourceMessageId ? store.get(b.sourceMessageId) : null;
2619
+ return (ma?.shardIndex ?? 0) - (mb?.shardIndex ?? 0);
2620
+ });
2621
+ // Build merged text content. Non-text blocks (rare in shards) are
2622
+ // preserved on the first shard's entry only.
2623
+ const mergedTextParts: string[] = [];
2624
+ const nonTextBlocks: ContentBlock[] = [];
2625
+ for (const r of sortedRun) {
2626
+ for (const block of r.content) {
2627
+ if (block.type === 'text') mergedTextParts.push(block.text);
2628
+ else nonTextBlocks.push(block);
2629
+ }
2630
+ }
2631
+ const mergedContent: ContentBlock[] = [
2632
+ ...nonTextBlocks,
2633
+ { type: 'text', text: mergedTextParts.join('') },
2634
+ ];
2635
+ out.push({
2636
+ index: out.length,
2637
+ sourceMessageId: sortedRun[0].sourceMessageId,
2638
+ sourceRelation: 'copy',
2639
+ participant: sortedRun[0].participant,
2640
+ content: mergedContent,
2641
+ });
2642
+ i = j;
2643
+ }
2644
+ return out;
2645
+ }
2646
+
2647
+ /** Get (lazily constructing) the configured picker instance. */
2648
+ protected getAdaptivePicker(): Picker {
2649
+ if (this._adaptivePicker) return this._adaptivePicker;
2650
+ const strategy: FoldingStrategy =
2651
+ this.config.foldingStrategy === 'oldest-first'
2652
+ ? new OldestFirstStrategy()
2653
+ : new FlatProfileStrategy();
2654
+ this._adaptivePicker = new Picker(strategy);
2655
+ return this._adaptivePicker;
2656
+ }
2657
+
2658
+ /**
2659
+ * Walk the summary tree to find the L_k ancestor of a message.
2660
+ * Returns null if no ancestor exists at that level (e.g., L_k not yet produced).
2661
+ *
2662
+ * Takes a pre-built summariesById map to avoid O(summaries) lookups per
2663
+ * call — for a chronicle with thousands of summaries and hundreds of
2664
+ * middle messages, the O(n) `find` would dominate compile latency.
2665
+ */
2666
+ protected findAncestorAt(
2667
+ messageId: MessageId,
2668
+ level: number,
2669
+ chunksByMessageId: ReadonlyMap<MessageId, Chunk>,
2670
+ summariesById?: ReadonlyMap<string, SummaryEntry>,
2671
+ ): SummaryEntry | null {
2672
+ if (level <= 0) return null;
2673
+ const chunk = chunksByMessageId.get(messageId);
2674
+ if (!chunk?.summaryId) return null;
2675
+ const lookup = (id: string): SummaryEntry | undefined =>
2676
+ summariesById ? summariesById.get(id) : this.summaries.find((s) => s.id === id);
2677
+ let current: SummaryEntry | undefined = lookup(chunk.summaryId);
2678
+ while (current && current.level < level) {
2679
+ const parentId = getSummaryParentId(current);
2680
+ if (!parentId) return null;
2681
+ current = lookup(parentId);
2682
+ }
2683
+ if (!current || current.level !== level) return null;
2684
+ return current;
2685
+ }
2686
+
2687
+ // ============================================================================
2688
+ // Hierarchical (threshold-driven) path
2689
+ // ============================================================================
2690
+
1233
2691
  /**
1234
2692
  * Select context entries using hierarchical compression with budget carryover.
1235
2693
  * Matches moltbot's budget waterfall: L3 → L2 → L1 with unused budget flowing down.
@@ -1506,23 +2964,113 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1506
2964
  // ============================================================================
1507
2965
 
1508
2966
  /**
1509
- * Build the compression instruction for an L1 chunk.
1510
- * Override in subclasses for phase-aware prompts.
2967
+ * Build the compression instruction for an L1 chunk in the hierarchical
2968
+ * path. Override in subclasses for domain-specific prompts (e.g.,
2969
+ * phase-aware prompts in KnowledgeStrategy).
2970
+ *
2971
+ * Default returns the KV-preserving first-person instruction matching
2972
+ * the hermes-autobio spec. The doc/reading-mode variant is exposed via
2973
+ * {@link getReadingChunkInstruction}.
1511
2974
  */
1512
2975
  protected getCompressionInstruction(chunk: Chunk, targetTokens: number): string {
1513
- return `Starting from my last message, please describe everything that has happened. Aim for about ${targetTokens} tokens. Describe it as you would to yourself, as if you are remembering what has happened.`;
2976
+ return formatInstruction(targetTokens);
2977
+ }
2978
+
2979
+ /**
2980
+ * Build the compression instruction for an L1 chunk that is part of a
2981
+ * substantially larger sharded message (reading mode). Override in
2982
+ * subclasses if domain logic needs to vary the reading-mode prompt.
2983
+ *
2984
+ * Default returns the reading-mode instruction that asks the model to
2985
+ * reflect on what reading was like rather than form a memory "of what
2986
+ * the chunk contained", which prevents voice drift into the content
2987
+ * author's perspective.
2988
+ */
2989
+ protected getReadingChunkInstruction(
2990
+ chunk: Chunk,
2991
+ totalTokens: number,
2992
+ targetTokens: number,
2993
+ ): string {
2994
+ return formatReadingChunkInstruction(totalTokens, targetTokens);
2995
+ }
2996
+
2997
+ /**
2998
+ * If the chunk is part of a substantially larger sharded message (total
2999
+ * bodyGroup tokens ≥ 2× the chunk's own tokens), return reading-context
3000
+ * metadata for the reading instruction. The 2× threshold means the
3001
+ * chunk represents a portion of something significantly larger — the
3002
+ * agent is reading, not conversing.
3003
+ *
3004
+ * Returns null when the chunk is a whole message (no bodyGroup), or
3005
+ * when bodyGroup total is < 2× chunk size (degenerate case — chunk
3006
+ * effectively IS the whole message). In those cases the standard
3007
+ * (non-reading) instruction is appropriate.
3008
+ */
3009
+ protected detectDocContext(
3010
+ chunk: Chunk,
3011
+ ctx: StrategyContext,
3012
+ ): { totalTokens: number; chunkTokens: number } | null {
3013
+ if (chunk.messages.length === 0) return null;
3014
+ const firstGroupId = chunk.messages[0].bodyGroupId;
3015
+ if (!firstGroupId) return null;
3016
+ // All messages in the chunk must share the same bodyGroupId
3017
+ for (const m of chunk.messages) {
3018
+ if (m.bodyGroupId !== firstGroupId) return null;
3019
+ }
3020
+ // Total tokens of the original message (sum of all shards in the bodyGroup).
3021
+ const allMessages = ctx.messageStore.getAll();
3022
+ let totalTokens = 0;
3023
+ for (const m of allMessages) {
3024
+ if (m.bodyGroupId === firstGroupId) {
3025
+ totalTokens += ctx.messageStore.estimateTokens(m);
3026
+ }
3027
+ }
3028
+ // Tokens in this chunk specifically.
3029
+ let chunkTokens = 0;
3030
+ for (const m of chunk.messages) {
3031
+ chunkTokens += ctx.messageStore.estimateTokens(m);
3032
+ }
3033
+ // Reading-mode threshold: the original message must be substantially
3034
+ // larger than this chunk. 2× means the chunk is at most half of the
3035
+ // whole — clearly a portion of something bigger.
3036
+ if (chunkTokens === 0 || totalTokens < 2 * chunkTokens) return null;
3037
+ return {
3038
+ totalTokens,
3039
+ chunkTokens,
3040
+ };
1514
3041
  }
1515
3042
 
1516
3043
  /**
1517
3044
  * Build the merge instruction for combining summaries into a higher level.
1518
3045
  * Override in subclasses for domain-specific merge prompts.
3046
+ *
3047
+ * Default returns the KV-preserving merge instruction. The reading-mode
3048
+ * variant (used when all leaves share a bodyGroup of substantial size)
3049
+ * is exposed via {@link getReadingMergeInstruction}.
1519
3050
  */
1520
3051
  protected getMergeInstruction(
1521
3052
  targetLevel: SummaryLevel,
1522
3053
  sources: SummaryEntry[],
1523
3054
  targetTokens: number
1524
3055
  ): string {
1525
- return `Please consolidate the memories since my last message into a single cohesive memory. Aim for about ${targetTokens} tokens. Write as you would to yourself — this is your autobiography, capturing the arc of what happened.`;
3056
+ const sourceLevelShown = sources.length > 0 ? Math.max(0, sources[0].level - 1) : 0;
3057
+ return formatMergeInstruction(targetLevel, sourceLevelShown, targetTokens);
3058
+ }
3059
+
3060
+ /**
3061
+ * Build the reading-mode merge instruction. Used when all leaf messages
3062
+ * underlying the merge share a bodyGroup of substantial size — the agent
3063
+ * has been reading a doc rather than conversing. Override in subclasses
3064
+ * if domain logic needs to vary the reading-mode merge prompt.
3065
+ */
3066
+ protected getReadingMergeInstruction(
3067
+ targetLevel: SummaryLevel,
3068
+ sources: SummaryEntry[],
3069
+ totalTokens: number,
3070
+ targetTokens: number,
3071
+ ): string {
3072
+ const sourceLevelShown = sources.length > 0 ? Math.max(0, sources[0].level - 1) : 0;
3073
+ return formatReadingMergeInstruction(targetLevel, sourceLevelShown, totalTokens, targetTokens);
1526
3074
  }
1527
3075
 
1528
3076
  /**