@animalabs/context-manager 0.5.4 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/src/adaptive/kv-control.d.ts +84 -33
  2. package/dist/src/adaptive/kv-control.d.ts.map +1 -1
  3. package/dist/src/adaptive/kv-control.js +510 -145
  4. package/dist/src/adaptive/kv-control.js.map +1 -1
  5. package/dist/src/adaptive/picker.d.ts +21 -0
  6. package/dist/src/adaptive/picker.d.ts.map +1 -1
  7. package/dist/src/adaptive/picker.js.map +1 -1
  8. package/dist/src/adaptive/strategies/kv-stable.d.ts +12 -4
  9. package/dist/src/adaptive/strategies/kv-stable.d.ts.map +1 -1
  10. package/dist/src/adaptive/strategies/kv-stable.js +67 -7
  11. package/dist/src/adaptive/strategies/kv-stable.js.map +1 -1
  12. package/dist/src/context-log.d.ts.map +1 -1
  13. package/dist/src/context-log.js +11 -0
  14. package/dist/src/context-log.js.map +1 -1
  15. package/dist/src/context-manager.d.ts +38 -6
  16. package/dist/src/context-manager.d.ts.map +1 -1
  17. package/dist/src/context-manager.js +51 -0
  18. package/dist/src/context-manager.js.map +1 -1
  19. package/dist/src/index.d.ts +4 -2
  20. package/dist/src/index.d.ts.map +1 -1
  21. package/dist/src/index.js +3 -0
  22. package/dist/src/index.js.map +1 -1
  23. package/dist/src/message-store.d.ts +60 -0
  24. package/dist/src/message-store.d.ts.map +1 -1
  25. package/dist/src/message-store.js +165 -10
  26. package/dist/src/message-store.js.map +1 -1
  27. package/dist/src/phase-channel.d.ts +13 -0
  28. package/dist/src/phase-channel.d.ts.map +1 -0
  29. package/dist/src/phase-channel.js +55 -0
  30. package/dist/src/phase-channel.js.map +1 -0
  31. package/dist/src/strategies/autobiographical.d.ts +247 -8
  32. package/dist/src/strategies/autobiographical.d.ts.map +1 -1
  33. package/dist/src/strategies/autobiographical.js +1419 -141
  34. package/dist/src/strategies/autobiographical.js.map +1 -1
  35. package/dist/src/strategies/index.d.ts +1 -1
  36. package/dist/src/strategies/index.d.ts.map +1 -1
  37. package/dist/src/strategies/index.js.map +1 -1
  38. package/dist/src/strategies/knowledge.d.ts +8 -0
  39. package/dist/src/strategies/knowledge.d.ts.map +1 -1
  40. package/dist/src/strategies/knowledge.js +23 -1
  41. package/dist/src/strategies/knowledge.js.map +1 -1
  42. package/dist/src/types/index.d.ts +1 -1
  43. package/dist/src/types/index.d.ts.map +1 -1
  44. package/dist/src/types/index.js.map +1 -1
  45. package/dist/src/types/strategy.d.ts +112 -11
  46. package/dist/src/types/strategy.d.ts.map +1 -1
  47. package/dist/src/types/strategy.js.map +1 -1
  48. package/dist/test/adaptive/doc-plus-chat.test.js +7 -3
  49. package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -1
  50. package/dist/test/adaptive/hard-fail.test.js +22 -0
  51. package/dist/test/adaptive/hard-fail.test.js.map +1 -1
  52. package/dist/test/adaptive/ingestion.test.js +3 -1
  53. package/dist/test/adaptive/ingestion.test.js.map +1 -1
  54. package/dist/test/adaptive/kv-control.test.js +17 -11
  55. package/dist/test/adaptive/kv-control.test.js.map +1 -1
  56. package/dist/test/adaptive/pin-at-level.test.d.ts +16 -0
  57. package/dist/test/adaptive/pin-at-level.test.d.ts.map +1 -0
  58. package/dist/test/adaptive/pin-at-level.test.js +133 -0
  59. package/dist/test/adaptive/pin-at-level.test.js.map +1 -0
  60. package/dist/test/adaptive/single-path-solve.test.d.ts +16 -0
  61. package/dist/test/adaptive/single-path-solve.test.d.ts.map +1 -0
  62. package/dist/test/adaptive/single-path-solve.test.js +204 -0
  63. package/dist/test/adaptive/single-path-solve.test.js.map +1 -0
  64. package/dist/test/anti-redundancy-budget.test.d.ts +17 -0
  65. package/dist/test/anti-redundancy-budget.test.d.ts.map +1 -0
  66. package/dist/test/anti-redundancy-budget.test.js +197 -0
  67. package/dist/test/anti-redundancy-budget.test.js.map +1 -0
  68. package/dist/test/chunk-persistence.test.d.ts +29 -0
  69. package/dist/test/chunk-persistence.test.d.ts.map +1 -0
  70. package/dist/test/chunk-persistence.test.js +290 -0
  71. package/dist/test/chunk-persistence.test.js.map +1 -0
  72. package/dist/test/compression-dedup.test.d.ts +20 -0
  73. package/dist/test/compression-dedup.test.d.ts.map +1 -0
  74. package/dist/test/compression-dedup.test.js +92 -0
  75. package/dist/test/compression-dedup.test.js.map +1 -0
  76. package/dist/test/compression-shape-invariants.test.js +40 -3
  77. package/dist/test/compression-shape-invariants.test.js.map +1 -1
  78. package/dist/test/head-window-order.test.d.ts +19 -0
  79. package/dist/test/head-window-order.test.d.ts.map +1 -0
  80. package/dist/test/head-window-order.test.js +159 -0
  81. package/dist/test/head-window-order.test.js.map +1 -0
  82. package/dist/test/image-strip-render-stats.test.js +25 -3
  83. package/dist/test/image-strip-render-stats.test.js.map +1 -1
  84. package/dist/test/knowledge-tool-boundary.test.d.ts +14 -0
  85. package/dist/test/knowledge-tool-boundary.test.d.ts.map +1 -0
  86. package/dist/test/knowledge-tool-boundary.test.js +93 -0
  87. package/dist/test/knowledge-tool-boundary.test.js.map +1 -0
  88. package/dist/test/l1-holdback.test.d.ts +15 -0
  89. package/dist/test/l1-holdback.test.d.ts.map +1 -0
  90. package/dist/test/l1-holdback.test.js +119 -0
  91. package/dist/test/l1-holdback.test.js.map +1 -0
  92. package/dist/test/merge-contiguity.test.d.ts +11 -0
  93. package/dist/test/merge-contiguity.test.d.ts.map +1 -0
  94. package/dist/test/merge-contiguity.test.js +74 -0
  95. package/dist/test/merge-contiguity.test.js.map +1 -0
  96. package/dist/test/message-store-window.test.d.ts +13 -0
  97. package/dist/test/message-store-window.test.d.ts.map +1 -0
  98. package/dist/test/message-store-window.test.js +177 -0
  99. package/dist/test/message-store-window.test.js.map +1 -0
  100. package/dist/test/pins.test.js +40 -0
  101. package/dist/test/pins.test.js.map +1 -1
  102. package/dist/test/speculation-cap.test.d.ts +9 -3
  103. package/dist/test/speculation-cap.test.d.ts.map +1 -1
  104. package/dist/test/speculation-cap.test.js +179 -3
  105. package/dist/test/speculation-cap.test.js.map +1 -1
  106. package/dist/test/summary-log-consistency.test.d.ts +18 -0
  107. package/dist/test/summary-log-consistency.test.d.ts.map +1 -0
  108. package/dist/test/summary-log-consistency.test.js +97 -0
  109. package/dist/test/summary-log-consistency.test.js.map +1 -0
  110. package/dist/test/tool-pairing-invariant.test.d.ts +15 -0
  111. package/dist/test/tool-pairing-invariant.test.d.ts.map +1 -0
  112. package/dist/test/tool-pairing-invariant.test.js +225 -0
  113. package/dist/test/tool-pairing-invariant.test.js.map +1 -0
  114. package/dist/tsconfig.tsbuildinfo +1 -1
  115. package/package.json +11 -3
  116. package/src/adaptive/kv-control.ts +554 -152
  117. package/src/adaptive/picker.ts +21 -0
  118. package/src/adaptive/strategies/kv-stable.ts +62 -12
  119. package/src/context-log.ts +11 -0
  120. package/src/context-manager.ts +61 -4
  121. package/src/index.ts +6 -2
  122. package/src/message-store.ts +202 -9
  123. package/src/phase-channel.ts +57 -0
  124. package/src/strategies/autobiographical.ts +1482 -167
  125. package/src/strategies/index.ts +1 -1
  126. package/src/strategies/knowledge.ts +25 -1
  127. package/src/types/index.ts +1 -0
  128. package/src/types/strategy.ts +120 -7
@@ -1,4 +1,5 @@
1
1
  import { NativeFormatter } from '@animalabs/membrane';
2
+ import { phaseChannel } from '../phase-channel.js';
2
3
  import { DEFAULT_AUTOBIOGRAPHICAL_CONFIG } from '../types/index.js';
3
4
  import { getSummaryParentId } from '../types/strategy.js';
4
5
  import { splitMixedToolMessages, stripUnpairedToolBlocks } from '../normalize-tool-messages.js';
@@ -53,7 +54,12 @@ function formatInstruction(targetTokens) {
53
54
  'Preserve concrete details — file paths, exact values, decisions, ' +
54
55
  `unresolved questions, the user\'s active asks. Target ~${targetTokens} ` +
55
56
  'tokens. Output only the memory body — no preamble, no section headers ' +
56
- 'unless they help preservation, no meta-commentary about summarizing.');
57
+ 'unless they help preservation, no meta-commentary about summarizing. ' +
58
+ 'Memorize only what actually happened in that slice: if it holds little ' +
59
+ 'beyond routine system traffic (heartbeats, empty turns, failure ' +
60
+ 'notices), a short memory saying so is correct — do not pad it by ' +
61
+ 're-narrating events you already remember from earlier as if they had ' +
62
+ 'just happened again.');
57
63
  }
58
64
  /**
59
65
  * Compression instruction for chunks that are part of a substantially larger
@@ -150,6 +156,53 @@ function safeSlice(str, start, end) {
150
156
  }
151
157
  return str.slice(start, end);
152
158
  }
159
+ /**
160
+ * Validate + normalize the optional V2 pin fold-depth bounds. Returns only the
161
+ * fields that are present and valid (non-negative integers), so a classic pin
162
+ * with no bounds persists exactly as before. `level` takes precedence over
163
+ * `maxLevel` (pin-at-k is stronger than a cap), so they're never both emitted.
164
+ */
165
+ function normalizePinLevels(opts) {
166
+ const clean = (v) => typeof v === 'number' && Number.isInteger(v) && v >= 0 ? v : undefined;
167
+ const level = clean(opts?.level);
168
+ if (level !== undefined)
169
+ return { level };
170
+ const maxLevel = clean(opts?.maxLevel);
171
+ return maxLevel !== undefined ? { maxLevel } : {};
172
+ }
173
+ /**
174
+ * Drop empty text blocks (`{type:'text', text:''}` or whitespace-only). The
175
+ * Anthropic API rejects them with 400 "text content blocks must be non-empty",
176
+ * which — thrown inside the speculative-compression drain — halts ALL
177
+ * compression. Non-text blocks (tool_use/tool_result/image) pass through
178
+ * unchanged, so tool pairing is preserved; callers drop any message left with
179
+ * an empty content array.
180
+ */
181
+ function stripEmptyTextBlocks(content) {
182
+ return content.filter((b) => {
183
+ if (b.type !== 'text')
184
+ return true;
185
+ const text = b.text;
186
+ return typeof text === 'string' && text.trim().length > 0;
187
+ });
188
+ }
189
+ /**
190
+ * Strip `thinking` / `redacted_thinking` blocks from compression INPUT.
191
+ *
192
+ * The summarizer must never be handed the agent's own reasoning:
193
+ * (a) signed thinking is only valid verbatim in the turn that produced it —
194
+ * replaying it into a rewritten summarize request corrupts the signature;
195
+ * (b) asking the model to summarize its own reasoning reads as reproducing /
196
+ * duplicating model output → `reasoning_extraction` refusal, which returns
197
+ * empty (→ "empty L1 summary, chunk left raw") or, worse, produces a
198
+ * summary that reproduces the reasoning as text — which then trips the
199
+ * SAME classifier on the MAIN thread once that summary is rendered.
200
+ * Thinking is scratch work, not history (the same rationale already applied to
201
+ * the summarizer's OUTPUT). Drop it from the input too.
202
+ */
203
+ function stripThinkingBlocks(content) {
204
+ return content.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking');
205
+ }
153
206
  /**
154
207
  * Autobiographical chunking strategy.
155
208
  * Compresses old conversation chunks into summaries in the model's own words.
@@ -167,6 +220,38 @@ export class AutobiographicalStrategy {
167
220
  pendingCompression = null;
168
221
  compressionQueue = [];
169
222
  _compressionCount = 0;
223
+ /**
224
+ * Monotonic counter of tick() operations that actually processed a queue item
225
+ * (compressed a chunk or executed a merge). `driveSpeculativeDrain` recurses
226
+ * while this advances — a length-delta check would falsely read "no progress"
227
+ * when a productive tick also enqueues a follow-on item (net queue length
228
+ * unchanged), halting the drain with work still queued.
229
+ */
230
+ _drainProgress = 0;
231
+ /**
232
+ * Chunks (keyed by their LAST message id — chunk membership is immutable
233
+ * once closed) whose L1 was explicitly demanded by a picker `produce` op.
234
+ * Demanded chunks bypass the `l1HoldbackChunks` window in `rebuildChunks`:
235
+ * speculation waits, demand doesn't. Entries become inert once the chunk
236
+ * compresses (compressed chunks are never re-queued), so no cleanup needed.
237
+ */
238
+ _demandedL1Chunks = new Set();
239
+ /**
240
+ * In-memory mirror of the persisted chunk records (autobio:chunks slot).
241
+ * Loaded in `loadPersistedState`, appended to when a chunk closes,
242
+ * updated by ID when its L1 lands.
243
+ */
244
+ chunkRecords = [];
245
+ chunkIdCounter = 0;
246
+ /**
247
+ * Fail-closed latch: set when most persisted records resolve to zero live
248
+ * messages (the messages-chain-break signature). While set, NO compression
249
+ * runs — duplicate memories are strictly worse than delayed compression.
250
+ */
251
+ chunkRecordsOrphaned = false;
252
+ _orphanWarned = false;
253
+ /** Record ids whose compression was refused by the L1 overlap guard. */
254
+ _overlapBlocked = new Set();
170
255
  // Hierarchical state
171
256
  summaries = [];
172
257
  summaryIdCounter = 0;
@@ -181,11 +266,13 @@ export class AutobiographicalStrategy {
181
266
  /** Namespace for state-id scoping. Set in `initialize()`. */
182
267
  ns = '';
183
268
  get summariesStateId() { return `${this.ns}/autobio:summaries`; }
269
+ get chunksStateId() { return `${this.ns}/autobio:chunks`; }
184
270
  get counterStateId() { return `${this.ns}/autobio:counter`; }
185
271
  get mergeQueueStateId() { return `${this.ns}/autobio:mergeQueue`; }
186
272
  get pinsStateId() { return `${this.ns}/autobio:pins`; }
187
273
  get resolutionsStateId() { return `${this.ns}/autobio:resolutions`; }
188
274
  get locksStateId() { return `${this.ns}/autobio:locks`; }
275
+ get calibrationStateId() { return `${this.ns}/autobio:calibration`; }
189
276
  /** Protected ranges (pins + documents). Loaded from chronicle in initialize. */
190
277
  pins = [];
191
278
  /** Monotonically increasing counter for pin ids. Persisted as part of the pins snapshot. */
@@ -322,7 +409,120 @@ export class AutobiographicalStrategy {
322
409
  break;
323
410
  }
324
411
  }
412
+ // Legacy stores (pre chunk-persistence) have L1 summaries but no chunk
413
+ // records — synthesize records from L1 sourceIds before the first
414
+ // rebuild, so covered ground is owned and never re-compressed.
415
+ this.migrateChunkRecords(ctx.messageStore);
325
416
  this.rebuildChunks(ctx.messageStore);
417
+ // Kick the merge ladder for pre-existing unmerged summaries. Normally a
418
+ // compression/merge completion does this, but a store that boots with a
419
+ // backlog above threshold and an empty queue (e.g. after a pyramid
420
+ // repair pruned duplicates and un-merged survivors) would otherwise
421
+ // never start consolidating. Idempotent: already-queued/merged sources
422
+ // are skipped.
423
+ if (this.config.hierarchical && !this.chunkRecordsOrphaned) {
424
+ this.checkMergeThreshold();
425
+ }
426
+ }
427
+ /**
428
+ * Whether chunk boundaries are persisted to the `autobio:chunks` slot and
429
+ * own the past. Subclasses with their own chunking (KnowledgeStrategy's
430
+ * semantic phases) opt out and keep the legacy recompute-every-rebuild
431
+ * behavior.
432
+ */
433
+ get chunkPersistenceEnabled() {
434
+ return true;
435
+ }
436
+ /**
437
+ * One-time lazy migration: a store with L1 summaries but an empty chunks
438
+ * slot predates chunk persistence. Each L1's sourceIds ARE the historical
439
+ * chunk boundary — synthesize a compressed record per L1, in message
440
+ * order. Stale generations from the old partial-tail compression bug
441
+ * (an L1 whose messages are ALL already covered by records synthesized so
442
+ * far — prefix families, same-range duplicates) get NO record; coverage is
443
+ * what blocks re-compression, and the repair tooling prunes their content
444
+ * separately.
445
+ */
446
+ migrateChunkRecords(store) {
447
+ if (!this.chunkPersistenceEnabled || !this.store)
448
+ return;
449
+ if (this.chunkRecords.length > 0)
450
+ return;
451
+ const l1s = this.summaries.filter(s => s.level === 1 && Array.isArray(s.sourceIds) && s.sourceIds.length > 0);
452
+ if (l1s.length === 0)
453
+ return;
454
+ const msgIndex = new Map();
455
+ store.getAll().forEach((m, i) => msgIndex.set(m.id, i));
456
+ // Sort by (start position asc, span length desc) so at each starting
457
+ // point the LONGEST generation claims the ground first.
458
+ const sorted = [...l1s].sort((a, b) => {
459
+ const sa = msgIndex.get(a.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
460
+ const sb = msgIndex.get(b.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
461
+ return sa - sb || b.sourceIds.length - a.sourceIds.length;
462
+ });
463
+ const covered = new Set();
464
+ let skippedStale = 0;
465
+ let skippedGhost = 0;
466
+ for (const s of sorted) {
467
+ // An L1 none of whose messages exist anymore can't own live ground.
468
+ if (!s.sourceIds.some(id => msgIndex.has(id))) {
469
+ skippedGhost++;
470
+ continue;
471
+ }
472
+ // Fully-covered = stale generation / contained duplicate.
473
+ if (s.sourceIds.every(id => covered.has(id))) {
474
+ skippedStale++;
475
+ continue;
476
+ }
477
+ for (const id of s.sourceIds)
478
+ covered.add(id);
479
+ this.appendChunkRecord({
480
+ id: `c-${this.chunkIdCounter++}`,
481
+ sourceIds: [...s.sourceIds],
482
+ compressed: true,
483
+ summaryId: s.id,
484
+ ...(s.phaseType ? { phaseType: s.phaseType } : {}),
485
+ });
486
+ }
487
+ console.warn(`[autobiographical] chunk-persistence migration: ${l1s.length} L1s → ` +
488
+ `${this.chunkRecords.length} records (${skippedStale} stale generations, ` +
489
+ `${skippedGhost} fully-orphaned L1s skipped)`);
490
+ }
491
+ /** Append a record to the chunks slot + in-memory mirror. */
492
+ appendChunkRecord(record) {
493
+ this.chunkRecords.push(record);
494
+ this.store?.appendToStateJson(this.chunksStateId, record);
495
+ }
496
+ /**
497
+ * Mark a chunk's record compressed, linking its L1. Resolves the log slot
498
+ * by ID against the persisted array — never by in-memory index (see
499
+ * setMergedInto for the clobber this avoids).
500
+ */
501
+ markChunkRecordCompressed(recordId, summaryId) {
502
+ if (!this.chunkPersistenceEnabled || !recordId)
503
+ return;
504
+ const rec = this.chunkRecords.find(r => r.id === recordId);
505
+ if (!rec)
506
+ return;
507
+ rec.compressed = true;
508
+ rec.summaryId = summaryId;
509
+ if (!this.store)
510
+ return;
511
+ const stored = this.store.getStateJson(this.chunksStateId);
512
+ if (!Array.isArray(stored))
513
+ return;
514
+ const payload = Buffer.from(JSON.stringify(rec));
515
+ let found = false;
516
+ for (let i = 0; i < stored.length; i++) {
517
+ const item = stored[i];
518
+ if (item && item.id === recordId) {
519
+ this.store.editStateItem(this.chunksStateId, i, payload);
520
+ found = true;
521
+ }
522
+ }
523
+ if (!found) {
524
+ console.warn(`[autobiographical] markChunkRecordCompressed: ${recordId} not found in persisted chunk log`);
525
+ }
326
526
  }
327
527
  /**
328
528
  * Register the three Chronicle state slots this strategy uses.
@@ -341,6 +541,17 @@ export class AutobiographicalStrategy {
341
541
  });
342
542
  }
343
543
  catch { /* already registered */ }
544
+ if (this.chunkPersistenceEnabled) {
545
+ try {
546
+ this.store.registerState({
547
+ id: this.chunksStateId,
548
+ strategy: 'append_log',
549
+ deltaSnapshotEvery: 50,
550
+ fullSnapshotEvery: 10,
551
+ });
552
+ }
553
+ catch { /* already registered */ }
554
+ }
344
555
  try {
345
556
  this.store.registerState({
346
557
  id: this.counterStateId,
@@ -362,6 +573,13 @@ export class AutobiographicalStrategy {
362
573
  });
363
574
  }
364
575
  catch { /* already registered */ }
576
+ try {
577
+ this.store.registerState({
578
+ id: this.calibrationStateId,
579
+ strategy: 'snapshot',
580
+ });
581
+ }
582
+ catch { /* already registered */ }
365
583
  // Adaptive-resolution state slots — only registered when the flag is on
366
584
  // so chronicles without the flag don't accumulate unused slots.
367
585
  if (this.config.adaptiveResolution) {
@@ -393,10 +611,48 @@ export class AutobiographicalStrategy {
393
611
  this.mergeQueue = [];
394
612
  this.pins = [];
395
613
  this.pinIdCounter = 0;
614
+ this.chunkRecords = [];
615
+ this.chunkIdCounter = 0;
396
616
  return;
397
617
  }
618
+ if (this.chunkPersistenceEnabled) {
619
+ const records = this.store.getStateJson(this.chunksStateId);
620
+ this.chunkRecords = (Array.isArray(records) ? records : [])
621
+ .filter(r => r && typeof r.id === 'string' && Array.isArray(r.sourceIds) && r.sourceIds.length > 0);
622
+ this.chunkIdCounter = this.chunkRecords.reduce((max, r) => {
623
+ const n = Number(r.id.replace(/^c-/, ''));
624
+ return Number.isFinite(n) && n >= max ? n + 1 : max;
625
+ }, 0);
626
+ this.chunkRecordsOrphaned = false;
627
+ this._orphanWarned = false;
628
+ this._overlapBlocked.clear();
629
+ }
398
630
  const summaries = this.store.getStateJson(this.summariesStateId);
399
- this.summaries = Array.isArray(summaries) ? summaries : [];
631
+ const loaded = Array.isArray(summaries) ? summaries : [];
632
+ // Drop empty-content summaries (bugged/empty generations from before the
633
+ // production guards). Recalling or merging one yields an empty text block →
634
+ // Anthropic 400 "content must be non-empty". Never let them re-enter memory.
635
+ const nonEmpty = loaded.filter(s => s && typeof s.content === 'string' && s.content.trim().length > 0);
636
+ const droppedEmpty = loaded.length - nonEmpty.length;
637
+ if (droppedEmpty > 0)
638
+ console.warn(`[autobiographical] dropped ${droppedEmpty} empty summary(ies) on load`);
639
+ // Dedupe by id, keeping the copy with mergedInto set (position of first
640
+ // occurrence preserved). Duplicate-id copies with diverging merge state
641
+ // exist in stores touched by the pre-fix setMergedInto index-desync bug;
642
+ // without dedupe, the plain copy stays on the unmerged frontier and its
643
+ // content renders twice (once itself, once via its parent's merge).
644
+ const byId = new Map();
645
+ for (const s of nonEmpty) {
646
+ const prev = byId.get(s.id);
647
+ if (!prev)
648
+ byId.set(s.id, s);
649
+ else if (!prev.mergedInto && s.mergedInto)
650
+ byId.set(s.id, s);
651
+ }
652
+ const dupes = nonEmpty.length - byId.size;
653
+ if (dupes > 0)
654
+ console.warn(`[autobiographical] deduped ${dupes} duplicate summary id(s) on load`);
655
+ this.summaries = [...byId.values()];
400
656
  const counter = this.store.getStateJson(this.counterStateId);
401
657
  this.summaryIdCounter = typeof counter === 'number' ? counter : 0;
402
658
  const queue = this.store.getStateJson(this.mergeQueueStateId);
@@ -475,6 +731,7 @@ export class AutobiographicalStrategy {
475
731
  kind: 'pin',
476
732
  name: opts?.name,
477
733
  created: Date.now(),
734
+ ...normalizePinLevels(opts),
478
735
  });
479
736
  this.persistPins();
480
737
  return id;
@@ -493,10 +750,20 @@ export class AutobiographicalStrategy {
493
750
  kind: 'document',
494
751
  name: opts?.name,
495
752
  created: Date.now(),
753
+ ...normalizePinLevels(opts),
496
754
  });
497
755
  this.persistPins();
498
756
  return id;
499
757
  }
758
+ /**
759
+ * V2 dynamic pin-at-level-k convenience: fix a range to render at EXACTLY
760
+ * fold level `level` (0 = raw). Honored only by `foldingStrategy: 'kv-stable'`;
761
+ * other strategies fall back to treating the range as raw. Equivalent to
762
+ * `pinRange(first, last, { level })`.
763
+ */
764
+ pinAtLevel(firstMessageId, lastMessageId, level, opts) {
765
+ return this.pinRange(firstMessageId, lastMessageId, { name: opts?.name, level });
766
+ }
500
767
  /** Remove a pin or document mark by id. Returns true if removed. */
501
768
  unpin(pinId) {
502
769
  const before = this.pins.length;
@@ -612,6 +879,45 @@ export class AutobiographicalStrategy {
612
879
  }
613
880
  return out;
614
881
  }
882
+ /**
883
+ * Resolve the V2 dynamic-pin fold-depth bounds (`ProtectedRange.level` /
884
+ * `maxLevel`) to message positions. Only pins that carry a bound appear; a
885
+ * classic raw pin (no bound) is absent here and handled by `pinnedPositions`.
886
+ * When ranges overlap, the FINEST requirement wins (lowest effective level):
887
+ * a fixed `level` clamps both ends; a `maxLevel` only caps depth. Honored
888
+ * solely by the KV-stable controller — see `ProtectedRange`.
889
+ */
890
+ pinLevelBounds(messages) {
891
+ const out = new Map();
892
+ if (this.pins.length === 0)
893
+ return out;
894
+ const positionOf = new Map();
895
+ for (let i = 0; i < messages.length; i++)
896
+ positionOf.set(messages[i].id, i);
897
+ for (const pin of this.pins) {
898
+ if (pin.level === undefined && pin.maxLevel === undefined)
899
+ continue;
900
+ const first = positionOf.get(pin.firstMessageId);
901
+ const last = positionOf.get(pin.lastMessageId);
902
+ if (first === undefined || last === undefined)
903
+ continue;
904
+ const lo = Math.min(first, last);
905
+ const hi = Math.max(first, last);
906
+ for (let i = lo; i <= hi; i++) {
907
+ const prev = out.get(i) ?? {};
908
+ // A fixed level is the strongest constraint; when two pins fix the same
909
+ // position, the shallower (lower) level wins (finest requirement).
910
+ if (pin.level !== undefined) {
911
+ prev.level = prev.level === undefined ? pin.level : Math.min(prev.level, pin.level);
912
+ }
913
+ else if (pin.maxLevel !== undefined) {
914
+ prev.maxLevel = prev.maxLevel === undefined ? pin.maxLevel : Math.min(prev.maxLevel, pin.maxLevel);
915
+ }
916
+ out.set(i, prev);
917
+ }
918
+ }
919
+ return out;
920
+ }
615
921
  /**
616
922
  * Append a summary to the in-memory list and to the chronicle AppendLog.
617
923
  * Single point so subclasses inherit persistence.
@@ -658,10 +964,29 @@ export class AutobiographicalStrategy {
658
964
  entry.mergedInto = mergedIntoId;
659
965
  if (!this.store)
660
966
  return;
661
- const index = this.summaries.indexOf(entry);
662
- if (index < 0)
967
+ // Resolve the log position by ID against the PERSISTED array — never by
968
+ // in-memory index. `loadPersistedState` filters empty-content summaries
969
+ // out of `this.summaries` while they remain in the log, so after a reload
970
+ // the in-memory index is shifted relative to the log slot. Editing by
971
+ // in-memory index wrote merge-updates onto NEIGHBORING entries, silently
972
+ // clobbering them (4 summaries lost in the 2026-07 Lena incident, leaving
973
+ // duplicate-id copies with diverging mergedInto). Update every stored
974
+ // copy with this id so past duplicates converge too.
975
+ const stored = this.store.getStateJson(this.summariesStateId);
976
+ if (!Array.isArray(stored))
663
977
  return;
664
- this.store.editStateItem(this.summariesStateId, index, Buffer.from(JSON.stringify(entry)));
978
+ let found = false;
979
+ const payload = Buffer.from(JSON.stringify(entry));
980
+ for (let i = 0; i < stored.length; i++) {
981
+ const item = stored[i];
982
+ if (item && item.id === entry.id) {
983
+ this.store.editStateItem(this.summariesStateId, i, payload);
984
+ found = true;
985
+ }
986
+ }
987
+ if (!found) {
988
+ console.warn(`[autobiographical] setMergedInto: ${entry.id} not found in persisted summary log — merge state not persisted`);
989
+ }
665
990
  }
666
991
  /**
667
992
  * Allocate the next summary-id counter value and persist the new counter.
@@ -755,6 +1080,11 @@ export class AutobiographicalStrategy {
755
1080
  for (const chunk of candidates) {
756
1081
  if (chunk.compressed)
757
1082
  continue;
1083
+ // Demand path: mark the chunk so the l1HoldbackChunks window in
1084
+ // rebuildChunks never filters it back out of the queue.
1085
+ const lastId = chunk.messages[chunk.messages.length - 1]?.id;
1086
+ if (lastId !== undefined)
1087
+ this._demandedL1Chunks.add(lastId);
758
1088
  if (this.compressionQueue.includes(chunk.index))
759
1089
  continue;
760
1090
  this.compressionQueue.push(chunk.index);
@@ -866,21 +1196,31 @@ export class AutobiographicalStrategy {
866
1196
  driveSpeculativeDrain(ctx) {
867
1197
  if (this.pendingCompression)
868
1198
  return;
869
- if (this.compressionQueue.length === 0 && this.mergeQueue.length === 0)
870
- return;
871
- if (this.isAtSpeculativeCap())
1199
+ // Merges consolidate existing L_k summaries into L_{k+1} and REDUCE the
1200
+ // unmerged-L1 count; L1 compression PRODUCES new unmerged L1s. The
1201
+ // speculation cap / preflight throttle *production* only — they must never
1202
+ // gate merges, otherwise exceeding the cap (e.g. after a manual backfill)
1203
+ // permanently deadlocks the drain: too many unmerged L1s trips the cap,
1204
+ // which blocks the very merges that would bring the count back down.
1205
+ const hasMerges = this.config.hierarchical === true && this.mergeQueue.length > 0;
1206
+ const hasCompression = this.compressionQueue.length > 0;
1207
+ if (!hasCompression && !hasMerges)
872
1208
  return;
873
- if (!this.shouldCompressPreflight())
1209
+ // Only bail when the *sole* available work is L1 compression that the cap
1210
+ // or preflight currently forbids. Merge work always proceeds.
1211
+ if (!hasMerges && (this.isAtSpeculativeCap() || !this.shouldCompressPreflight()))
874
1212
  return;
875
- const beforeChunks = this.compressionQueue.length;
876
- const beforeMerges = this.mergeQueue.length;
1213
+ const progressBefore = this._drainProgress;
877
1214
  this.tick(ctx)
878
1215
  .then(() => {
879
- const afterChunks = this.compressionQueue.length;
880
- const afterMerges = this.mergeQueue.length;
881
- // Made progress if either queue shrank.
882
- const progressed = afterChunks < beforeChunks || afterMerges < beforeMerges;
883
- if (!progressed)
1216
+ // Progress = the tick actually processed a queue item (compress or
1217
+ // merge), tracked by `_drainProgress`. A queue-length delta is the
1218
+ // wrong signal: a productive merge tick can also enqueue a follow-on
1219
+ // merge, leaving the length unchanged which the old check misread as
1220
+ // "no progress" and halted the drain mid-backlog. A genuine no-op tick
1221
+ // (empty queues, at-cap with no merges, no membrane) doesn't advance
1222
+ // the counter, so this still stops cleanly (no runaway recursion).
1223
+ if (this._drainProgress === progressBefore)
884
1224
  return;
885
1225
  // Recurse to drain more. queueMicrotask defers until the current
886
1226
  // task is done, letting other code (the agent's stream consumer)
@@ -892,15 +1232,24 @@ export class AutobiographicalStrategy {
892
1232
  });
893
1233
  }
894
1234
  /**
895
- * Whether the strategy's pending+queued L1 budget has reached the cap
1235
+ * Whether the count of *produced, unmerged* L1 summaries has reached the cap
896
1236
  * configured by `maxSpeculativeL1s`. If no cap is set, always false.
1237
+ *
1238
+ * The cap bounds how many L1 summaries may sit un-consolidated before the
1239
+ * strategy must merge them into L_{k+1} (bounding prefix churn / merge debt).
1240
+ * It deliberately does NOT count the pending `compressionQueue`: that queue is
1241
+ * the backlog of work to be *drained*, not produced summaries. Counting it
1242
+ * here would let a large backlog permanently trip the cap and block the very
1243
+ * compression that would clear it — a deadlock (merges relieve the cap, but
1244
+ * compression of the backlog never resumes). The throttle is on produced L1s;
1245
+ * the queue drains freely, with merges keeping the unmerged count under the cap.
897
1246
  */
898
1247
  isAtSpeculativeCap() {
899
1248
  const cap = this.config.maxSpeculativeL1s;
900
1249
  if (cap === undefined || cap < 0)
901
1250
  return false;
902
1251
  const unmergedL1s = this.summaries.filter(s => s.level === 1 && !s.mergedInto).length;
903
- return unmergedL1s + this.compressionQueue.length > cap;
1252
+ return unmergedL1s > cap;
904
1253
  }
905
1254
  /**
906
1255
  * Preflight hook for whether speculative compression should fire on
@@ -912,15 +1261,20 @@ export class AutobiographicalStrategy {
912
1261
  return true;
913
1262
  }
914
1263
  async tick(ctx) {
1264
+ phaseChannel.report('compress-tick'); // liveness-watchdog phase
915
1265
  if (this.pendingCompression)
916
1266
  return;
917
1267
  if (!ctx.membrane) {
918
1268
  console.warn('AutobiographicalStrategy: No membrane instance for compression');
919
1269
  return;
920
1270
  }
921
- // Priority 1: Compress raw chunks → L1
922
- if (this.compressionQueue.length > 0) {
1271
+ // Priority 1: Compress raw chunks → L1. Skipped while at the speculative
1272
+ // cap (maxSpeculativeL1s) so we don't pile up more unmerged L1s; the merge
1273
+ // priority below still runs to consolidate existing L1s and relieve the cap.
1274
+ // No cap configured → isAtSpeculativeCap() is always false → unchanged.
1275
+ if (this.compressionQueue.length > 0 && !this.isAtSpeculativeCap()) {
923
1276
  const chunkIndex = this.compressionQueue.shift();
1277
+ this._drainProgress++; // consumed a queue item (real work or stale-cleanup)
924
1278
  const chunk = this.chunks[chunkIndex];
925
1279
  if (!chunk || chunk.compressed)
926
1280
  return;
@@ -944,6 +1298,8 @@ export class AutobiographicalStrategy {
944
1298
  // and the next tick() retries it.
945
1299
  if (this.config.hierarchical && this.mergeQueue.length > 0) {
946
1300
  const merge = this.mergeQueue[0];
1301
+ this._drainProgress++; // executing a merge is real work, even if a
1302
+ // follow-on merge gets enqueued and the queue length nets out unchanged
947
1303
  this.pendingCompression = this.executeMerge(merge.level, merge.sourceIds, ctx);
948
1304
  try {
949
1305
  await this.pendingCompression;
@@ -1032,6 +1388,14 @@ export class AutobiographicalStrategy {
1032
1388
  // snapshot, so it reflects what the last compile actually rendered rather than
1033
1389
  // re-deriving the full pyramid (which, under adaptive resolution, bears little
1034
1390
  // resemblance to the folded output).
1391
+ //
1392
+ // CAVEAT: the final structural passes (`trimOrphanedToolUse`,
1393
+ // `enforceToolPairing`) run AFTER the per-entry tallies and BEFORE `rsEnd()`,
1394
+ // and are NOT reflected in the snapshot. Trims only remove entries (total
1395
+ // over-counts by at most the trimmed tail); the pairing validator can also
1396
+ // ADD stub tool_result entries (total then under-counts by those stubs).
1397
+ // Both deltas are a handful of tokens — the stats describe the selection, not
1398
+ // the byte-exact wire payload. Do not treat `total` as an exact wire count.
1035
1399
  // ===========================================================================
1036
1400
  _rs = null;
1037
1401
  _lastRenderStats = null;
@@ -1145,16 +1509,20 @@ export class AutobiographicalStrategy {
1145
1509
  * chronological order, dropping a leading orphan tool_result if its tool_use
1146
1510
  * fell into the evicted older portion.
1147
1511
  */
1148
- emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokensBefore) {
1512
+ emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokensBefore,
1513
+ // Post-strip per-message estimates (2026-07-12): eviction must price a
1514
+ // message the way the stripped render will. Raw pricing counted every
1515
+ // stripped image at full cost and evicted most of an image-era tail the
1516
+ // recent-window walk had correctly admitted (mythos: 601-message tail
1517
+ // admitted, 228 survived eviction, 45k rendered of a 120k window).
1518
+ pse) {
1149
1519
  if (recentStart >= messages.length)
1150
1520
  return { messages: 0, tokens: 0 };
1521
+ const est = (i) => pse?.[i] ?? store.estimateTokens(messages[i]);
1151
1522
  const accepted = [];
1152
1523
  let acceptedTokens = 0;
1153
1524
  for (let i = messages.length - 1; i >= recentStart; i--) {
1154
- const msg = messages[i];
1155
- const tokens = msgCap > 0
1156
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1157
- : store.estimateTokens(msg);
1525
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1158
1526
  if (this.isOverBudget(totalTokensBefore + acceptedTokens + tokens, maxTokens))
1159
1527
  break;
1160
1528
  accepted.push(i);
@@ -1171,9 +1539,7 @@ export class AutobiographicalStrategy {
1171
1539
  for (const i of accepted) {
1172
1540
  const msg = messages[i];
1173
1541
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
1174
- const tokens = msgCap > 0
1175
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1176
- : store.estimateTokens(msg);
1542
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1177
1543
  entries.push({
1178
1544
  index: entries.length,
1179
1545
  sourceMessageId: msg.id,
@@ -1216,31 +1582,186 @@ export class AutobiographicalStrategy {
1216
1582
  * No system prompt — framing via message structure only.
1217
1583
  */
1218
1584
  async compressChunkHierarchical(chunk, ctx) {
1585
+ phaseChannel.report('compress-chunk'); // liveness-watchdog phase
1219
1586
  if (!ctx.membrane) {
1220
1587
  throw new Error('No membrane instance for compression');
1221
1588
  }
1589
+ // ---- Duplicate-formation guards (layered) ----
1590
+ // Merged from two independent fixes for the same disease:
1591
+ //
1592
+ // 1. EXACT MATCH → adopt (bug 6.10, Tengro). rebuildChunks (fired by
1593
+ // onNewMessage / select) can re-queue a span whose compression already
1594
+ // completed against a stale chunk object — or, under chunk
1595
+ // persistence, a crash between the L1 append and the record edit
1596
+ // leaves a record uncompressed with its summary already in the log.
1597
+ // Adopt the existing summary, skip the LLM call, and heal the record.
1598
+ const chunkIdKey = this.chunkKey(chunk);
1599
+ const exactExisting = this.summaries.find(s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey);
1600
+ if (exactExisting) {
1601
+ chunk.compressed = true;
1602
+ chunk.summaryId = exactExisting.id;
1603
+ this.markChunkRecordCompressed(chunk.recordId, exactExisting.id);
1604
+ return;
1605
+ }
1606
+ const coveredByL1 = new Set();
1607
+ for (const s of this.summaries) {
1608
+ if (s.level === 1 && Array.isArray(s.sourceIds)) {
1609
+ for (const id of s.sourceIds)
1610
+ coveredByL1.add(id);
1611
+ }
1612
+ }
1613
+ // 2. FULLY COVERED (non-exact) → drop rather than duplicate history
1614
+ // (bug 6.10, Tengro): boundaries shifted across a rebuild and every
1615
+ // message is already inside some L1. Marking compressed WITHOUT a
1616
+ // summaryId means the uncompressed-middle fallback skips these
1617
+ // messages — they render only via the covering L1s. Safe while
1618
+ // `recentStart` advances monotonically (a fully-covered OLD chunk
1619
+ // can't intersect the recent-exclusion window); if that assumption is
1620
+ // ever weakened, reinstate a raw fallback (chunk-level `coveredBy`)
1621
+ // rather than dropping. The chunk record (if any) is deliberately
1622
+ // left uncompressed as an operator breadcrumb.
1623
+ if (chunk.messages.length > 0 && chunk.messages.every(m => coveredByL1.has(m.id))) {
1624
+ console.warn(`[autobiographical] dedup guard: chunk ${chunk.recordId ?? `#${chunk.index}`} ` +
1625
+ `is fully covered by existing L1s under different boundaries — dropped, not re-compressed.`);
1626
+ chunk.compressed = true;
1627
+ return;
1628
+ }
1629
+ // 3. PARTIAL OVERLAP → refuse (strict; chunk persistence). With
1630
+ // close-then-compress there is NO legitimate way for a chunk to
1631
+ // partially overlap an existing L1's span. If it happens anyway
1632
+ // (bookkeeping bug, store surgery gone wrong), refuse to produce:
1633
+ // a warning in the log is strictly better than a duplicate memory
1634
+ // in an agent's head.
1635
+ const overlapIds = chunk.messages.filter(m => coveredByL1.has(m.id)).map(m => m.id);
1636
+ if (overlapIds.length > 0) {
1637
+ const key = chunk.recordId ?? chunkIdKey;
1638
+ if (!this._overlapBlocked.has(key)) {
1639
+ this._overlapBlocked.add(key);
1640
+ console.error(`[autobiographical] OVERLAP GUARD: refusing to compress chunk ` +
1641
+ `${chunk.recordId ?? `#${chunk.index}`} — ${overlapIds.length}/${chunk.messages.length} ` +
1642
+ `of its messages are already covered by existing L1 summaries ` +
1643
+ `(first: ${overlapIds[0]}). Duplicate-memory formation blocked; ` +
1644
+ `investigate before resuming this span.`);
1645
+ }
1646
+ return;
1647
+ }
1222
1648
  const targetTokens = this.config.summaryTargetTokens ?? 2000;
1223
1649
  const agentParticipant = this.config.summaryParticipant ?? 'Claude';
1650
+ // ---- 0. Thin-chunk guard ----
1651
+ // A chunk of silent/skip turns and bare system traffic gives the
1652
+ // summarizer nothing to remember. Asked anyway, it confabulates: it
1653
+ // reaches for the nearest salient content (head window, prior recall
1654
+ // pairs) and narrates it as if it just happened — each such L1 then
1655
+ // compounds through merges (the "68 initiations" incident). Store a
1656
+ // mechanical stub without an LLM call instead. Chunks with any
1657
+ // non-text blocks (tool cycles, images) are never stubbed.
1658
+ const minChunkChars = this.config.minChunkCharsForLLM ?? 200;
1659
+ if (minChunkChars > 0) {
1660
+ let substantiveChars = 0;
1661
+ let hasNonText = false;
1662
+ for (const m of chunk.messages) {
1663
+ for (const b of m.content) {
1664
+ if (b.type === 'text')
1665
+ substantiveChars += b.text.trim().length;
1666
+ else
1667
+ hasNonText = true;
1668
+ }
1669
+ }
1670
+ if (!hasNonText && substantiveChars < minChunkChars) {
1671
+ const messageIds = chunk.messages.map(m => m.id);
1672
+ const stub = {
1673
+ id: `L1-${this.nextSummaryIdCounter()}`,
1674
+ level: 1,
1675
+ content: `(A quiet stretch: ${chunk.messages.length} messages of routine ` +
1676
+ `system traffic — heartbeats, empty turns, notices. Nothing ` +
1677
+ `happened worth remembering.)`,
1678
+ tokens: 40,
1679
+ sourceLevel: 0,
1680
+ sourceIds: messageIds,
1681
+ sourceRange: { first: messageIds[0], last: messageIds[messageIds.length - 1] },
1682
+ created: Date.now(),
1683
+ phaseType: chunk.phaseType,
1684
+ };
1685
+ this.pushSummary(stub);
1686
+ chunk.compressed = true;
1687
+ chunk.summaryId = stub.id;
1688
+ this.markChunkRecordCompressed(chunk.recordId, stub.id);
1689
+ this._compressionCount++;
1690
+ logCompressionCall({
1691
+ operation: 'compress_l1',
1692
+ system: null,
1693
+ messages: [],
1694
+ metadata: {
1695
+ stub: true,
1696
+ chunk_message_ids: messageIds,
1697
+ chunk_size: chunk.messages.length,
1698
+ substantive_chars: substantiveChars,
1699
+ min_chunk_chars: minChunkChars,
1700
+ summary_id: stub.id,
1701
+ },
1702
+ response: stub.content,
1703
+ });
1704
+ this.checkMergeThreshold();
1705
+ return;
1706
+ }
1707
+ }
1224
1708
  // Build the KV-preserving prompt per hermes-autobio spec:
1225
1709
  //
1226
- // 1. Prior summaries narrativized as CM-asks / agent-recalls
1710
+ // 1. Headthe raw chronicle opening (identity anchor), FIRST,
1711
+ // exactly where the original instance saw it. It MUST precede
1712
+ // the recall pairs: when it followed them (pre-2026-07 order),
1713
+ // it read as the most recent live conversation, and for thin
1714
+ // chunks the summarizer narrated the head as fresh events
1715
+ // ("Antra came to me to explore the transformation story
1716
+ // again…"), compounding across merges into runaway false
1717
+ // memories (the "68 initiations" incident). Chronological
1718
+ // order is also the KV-stable order — the head never changes.
1719
+ // (executeMerge always had head-first; this site was the odd
1720
+ // one out.)
1721
+ // 2. Prior summaries — narrativized as CM-asks / agent-recalls
1227
1722
  // pairs, in source order. The unmerged frontier of the
1228
1723
  // summary forest: any summary that has not yet been merged
1229
1724
  // into a higher level. After merges run, the L_{k+1} replaces
1230
1725
  // its L_k children — using the children plus their parent
1231
1726
  // doubles the prompt size unboundedly.
1232
- // 2. Headraw messages before the chunk that aren't already
1233
- // represented by a prior summary.
1234
- // 3. Marker — in-band signal that a memory is about to form.
1235
- // 4. Chunk — raw messages being compressed, as the agent
1727
+ // 3. Raw middle — messages between head and chunk not covered by
1728
+ // any summary (usually empty).
1729
+ // 4. Marker — in-band signal that a memory is about to form.
1730
+ // 5. Chunk — raw messages being compressed, as the agent
1236
1731
  // experienced them.
1237
- // 5. Instruction — doc-aware if the chunk is part of a bodyGroup.
1732
+ // 6. Instruction — doc-aware if the chunk is part of a bodyGroup.
1238
1733
  //
1239
1734
  // There is intentionally NO tail_after_chunk: that would leak
1240
1735
  // future information into the model's KV state and corrupt the
1241
1736
  // as-of framing of memory formation.
1242
1737
  const llmMessages = [];
1243
- // ---- 1. Prior recall pairs ----
1738
+ // ---- 1. Head window (raw, ALWAYS present, FIRST) ----
1739
+ //
1740
+ // The head is the foundational identity anchor: the actual opening
1741
+ // of the chronicle (the user's first message, the agent's first
1742
+ // reply, the system context if any). It establishes WHO is speaking
1743
+ // to WHOM. Without it, when the chunk content is heavily first-person
1744
+ // from someone other than the agent (e.g., a user-shared document),
1745
+ // the agent loses its first-person grounding and drifts into the
1746
+ // content author's voice.
1747
+ //
1748
+ // The head is the configured head window — not "everything before
1749
+ // the chunk." For doc-heavy chronicles, "everything before" would
1750
+ // be hundreds of thousands of tokens; the recall pairs below
1751
+ // represent that intermediate content. The head is just the
1752
+ // permanent prefix that the original instance always saw.
1753
+ //
1754
+ // Head messages are excluded from compression by `getCompressibleMessages`
1755
+ // (they're outside the chunking range), so they won't appear in
1756
+ // any L1's sourceIds — no overlap with the recall pairs below.
1757
+ const allMessages = ctx.messageStore.getAll();
1758
+ const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1759
+ const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1760
+ for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1761
+ const m = allMessages[i];
1762
+ llmMessages.push({ participant: m.participant, content: m.content });
1763
+ }
1764
+ // ---- 2. Prior recall pairs ----
1244
1765
  // Filter to the unmerged frontier: any summary whose `mergedInto`
1245
1766
  // is unset. After merge, the children's mergedInto points at the
1246
1767
  // parent and the parent stands alone with that source range. The
@@ -1249,7 +1770,11 @@ export class AutobiographicalStrategy {
1249
1770
  // converged to ~500 L1s that never aged out, blowing the 200k
1250
1771
  // window around chunk 118.
1251
1772
  const priorSummaries = this.summaries
1252
- .filter((s) => !s.mergedInto)
1773
+ // Skip empty-content summaries: emitting `{type:'text', text:''}` as a
1774
+ // recall pair triggers Anthropic 400 "text content blocks must be
1775
+ // non-empty", which stalls ALL compression (mirrors the render-path guard
1776
+ // + load-drop). A single empty summary otherwise poisons every compression.
1777
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1253
1778
  .sort((a, b) => a.sourceRange.first.localeCompare(b.sourceRange.first));
1254
1779
  // Token-budget cap (see capRecallPairs). Defense-in-depth: even with
1255
1780
  // merged exclusion the unmerged frontier can be large at extreme scale.
@@ -1278,32 +1803,7 @@ export class AutobiographicalStrategy {
1278
1803
  content: [{ type: 'text', text: s.content }],
1279
1804
  });
1280
1805
  }
1281
- // ---- 2. Head window (raw, ALWAYS present) ----
1282
- //
1283
- // The head is the foundational identity anchor: the actual opening
1284
- // of the chronicle (the user's first message, the agent's first
1285
- // reply, the system context if any). It establishes WHO is speaking
1286
- // to WHOM. Without it, when the chunk content is heavily first-person
1287
- // from someone other than the agent (e.g., a user-shared document),
1288
- // the agent loses its first-person grounding and drifts into the
1289
- // content author's voice.
1290
- //
1291
- // The head is the configured head window — not "everything before
1292
- // the chunk." For doc-heavy chronicles, "everything before" would
1293
- // be hundreds of thousands of tokens; the recall pairs already
1294
- // represent that intermediate content. The head is just the
1295
- // permanent prefix that the original instance always saw.
1296
- //
1297
- // Head messages are excluded from compression by `getCompressibleMessages`
1298
- // (they're outside the chunking range), so they won't appear in
1299
- // any L1's sourceIds — no overlap with the recall pairs above.
1300
- const allMessages = ctx.messageStore.getAll();
1301
- const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1302
- const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1303
- for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1304
- const m = allMessages[i];
1305
- llmMessages.push({ participant: m.participant, content: m.content });
1306
- }
1806
+ // ---- 3. Raw middle ----
1307
1807
  // Any raw messages between the head and the chunk that aren't yet
1308
1808
  // represented by any summary — usually empty in adaptive-resolution
1309
1809
  // mode, since chunking proceeds contiguously and summaries cover
@@ -1337,16 +1837,16 @@ export class AutobiographicalStrategy {
1337
1837
  llmMessages.push({ participant: m.participant, content: m.content });
1338
1838
  }
1339
1839
  }
1340
- // ---- 3. In-band marker ----
1840
+ // ---- 4. In-band marker ----
1341
1841
  llmMessages.push({
1342
1842
  participant: 'Context Manager',
1343
1843
  content: [{ type: 'text', text: COMPRESSION_MARKER }],
1344
1844
  });
1345
- // ---- 4. Chunk messages raw ----
1845
+ // ---- 5. Chunk messages raw ----
1346
1846
  for (const m of chunk.messages) {
1347
1847
  llmMessages.push({ participant: m.participant, content: m.content });
1348
1848
  }
1349
- // ---- 5. Instruction (reading-mode aware) ----
1849
+ // ---- 6. Instruction (reading-mode aware) ----
1350
1850
  //
1351
1851
  // When the chunk is a portion of a substantially larger sharded message
1352
1852
  // (≥ 2× chunk size), use the reading-mode instruction. It avoids the
@@ -1376,6 +1876,16 @@ export class AutobiographicalStrategy {
1376
1876
  // case). The API rejects any tool_use that isn't immediately followed
1377
1877
  // by its tool_result, and any tool_result that doesn't follow a use.
1378
1878
  const cleaned = stripUnpairedToolBlocks(collapsed);
1879
+ // Without the agent's live tool definitions, a request that replays
1880
+ // tool-block-bearing history is deterministically refused
1881
+ // (reasoning_extraction -- see the `tools` comment below). Tools are
1882
+ // pushed by the host on every activation; before the first activation
1883
+ // of a session, defer rather than burn a doomed full-window call.
1884
+ const chunkHasToolBlocks = cleaned.some(m => m.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'));
1885
+ if (chunkHasToolBlocks && !(ctx.tools && ctx.tools.length > 0)) {
1886
+ console.warn('[autobiographical] deferring chunk compression: history contains tool blocks but host has not provided tool definitions yet (ctx.tools empty) — will retry after next activation');
1887
+ return;
1888
+ }
1379
1889
  // NO system prompt. The agent's identity is established by the head
1380
1890
  // (the actual conversation opening — user message + agent reply that
1381
1891
  // grounded the original instance). A system prompt would (a) add a
@@ -1385,11 +1895,41 @@ export class AutobiographicalStrategy {
1385
1895
  // structural one carried by the conversation itself. Anchoring
1386
1896
  // identity by the chronicle's actual head is more honest.
1387
1897
  const request = {
1388
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
1898
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
1899
+ // raw history that can carry more inline image bytes than the API's
1900
+ // request cap. Dropping the OLDEST images from the summarizer's view is
1901
+ // acceptable policy here — the summary describes the span, it does not
1902
+ // preserve pixels — and membrane error-logs every exercised shed. All
1903
+ // other callers fail loudly instead (no silent transport mutation).
1904
+ shedOversizeImages: true,
1905
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
1906
+ // message left with no content. An empty-content turn (e.g. a silent/skip
1907
+ // turn that produced no text) otherwise reaches the API as an empty text
1908
+ // block → 400 "text content blocks must be non-empty", which throws in the
1909
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
1910
+ // recall-header guard — together they cover every source of the 400.)
1911
+ messages: cleaned
1912
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
1913
+ .filter(m => m.content.length > 0),
1389
1914
  config: {
1390
1915
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1391
- maxTokens: Math.round(targetTokens * 1.5),
1916
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
1917
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
1918
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
1919
+ // off rich memories (stop=max_tokens).
1920
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
1392
1921
  },
1922
+ // Declare the agent's live tools. A summarizer request that replays
1923
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
1924
+ // safety classifier as a foreign agent trace being duplicated ->
1925
+ // deterministic reasoning_extraction refusal of every memory-write
1926
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
1927
+ // tools the live instance runs with is also strictly MORE faithful to
1928
+ // the original context, so it is the KV-honest choice, not a
1929
+ // workaround. Undefined before the first activation of a session --
1930
+ // acceptable: those chunks stay raw and are retried after the agent's
1931
+ // first turn (see the defer guard in compressChunkHierarchical).
1932
+ tools: ctx.tools,
1393
1933
  };
1394
1934
  const callStart = Date.now();
1395
1935
  let logResponse;
@@ -1406,12 +1946,40 @@ export class AutobiographicalStrategy {
1406
1946
  .map(b => b.text)
1407
1947
  .join('\n');
1408
1948
  logResponse = summaryText;
1949
+ // A bugged/empty generation (summarizer returned no text — spent budget on
1950
+ // thinking, truncated, etc.) must NOT be stored: recalled later it becomes
1951
+ // an empty assistant text block → Anthropic 400 "content must be non-empty".
1952
+ // Leave the chunk raw rather than poisoning memory with an empty summary.
1953
+ if (!summaryText.trim()) {
1954
+ console.warn(`[autobiographical] empty L1 summary for chunk of ${chunk.messages.length} msgs — skipping (chunk left raw)`);
1955
+ return;
1956
+ }
1957
+ // Re-check the dedup guard AFTER the await: summary state may have
1958
+ // changed while the LLM call was in flight (persisted-state reload,
1959
+ // or any future concurrent producer). Discarding a paid-for result
1960
+ // is cheaper than storing a duplicate L1 over the same messages.
1961
+ const postExisting = this.summaries.find(s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey);
1962
+ if (postExisting) {
1963
+ chunk.compressed = true;
1964
+ chunk.summaryId = postExisting.id;
1965
+ return;
1966
+ }
1409
1967
  const messageIds = chunk.messages.map(m => m.id);
1410
1968
  const entry = {
1411
1969
  id: `L1-${this.nextSummaryIdCounter()}`,
1412
1970
  level: 1,
1413
1971
  content: summaryText,
1414
- tokens: Math.ceil(summaryText.length / 4),
1972
+ // Exact when available (2026-07-12): the compression response's own
1973
+ // usage.outputTokens IS the true token count of the text it just
1974
+ // wrote — the single most-reused number in the pyramid (fold floor,
1975
+ // middle budget, recall caps). Estimate only as fallback.
1976
+ tokens: response.usage?.outputTokens &&
1977
+ response.usage.outputTokens > 0 &&
1978
+ // outputTokens includes scratch thinking when present — only exact
1979
+ // when the whole response is the summary text itself.
1980
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
1981
+ ? response.usage.outputTokens
1982
+ : Math.ceil(summaryText.length / 3),
1415
1983
  sourceLevel: 0,
1416
1984
  sourceIds: messageIds,
1417
1985
  sourceRange: {
@@ -1424,6 +1992,7 @@ export class AutobiographicalStrategy {
1424
1992
  this.pushSummary(entry);
1425
1993
  chunk.compressed = true;
1426
1994
  chunk.summaryId = entry.id;
1995
+ this.markChunkRecordCompressed(chunk.recordId, entry.id);
1427
1996
  this._compressionCount++;
1428
1997
  logSummaryId = entry.id;
1429
1998
  this.checkMergeThreshold();
@@ -1473,7 +2042,63 @@ export class AutobiographicalStrategy {
1473
2042
  * already-eligible siblings, producing N near-identical higher-level
1474
2043
  * summaries when the queue eventually drains.
1475
2044
  */
2045
+ /**
2046
+ * Pick merge sources as the oldest CONTIGUOUS run (2026-07-12 fix).
2047
+ *
2048
+ * The old rule merged "whatever N are unmerged" in creation order. On a
2049
+ * store whose frontier held both June L2s and a July L2, that minted an
2050
+ * L3 spanning months of already-merged history (mythos L3-415, 0-3853 of
2051
+ * 3995 messages) — a group whose range straddles the recent window can
2052
+ * never fold (group-atomicity vs the raw zone), so the whole deep lineage
2053
+ * above it went unusable and the fold floor stopped fitting the budget.
2054
+ *
2055
+ * Rules: order candidates by live source position; break runs where the
2056
+ * positional gap exceeds `mergeContiguityGapLimit` (holes from wiped/
2057
+ * pruned nodes are fine, cross-era bridges are not); exclude candidates
2058
+ * whose OWN span exceeds `mergeMaxSourceSpanMessages` (replay-era wide-
2059
+ * span summaries would bridge anything they join); merge the oldest run
2060
+ * that still has `threshold` members.
2061
+ */
2062
+ contiguousMergeCandidates(unmerged, threshold) {
2063
+ if (unmerged.length < threshold)
2064
+ return null;
2065
+ const messageOrder = new Map();
2066
+ let seq = 0;
2067
+ for (const ch of this.chunks) {
2068
+ for (const m of ch.messages)
2069
+ messageOrder.set(m.id, seq++);
2070
+ }
2071
+ const gapLimit = this.config.mergeContiguityGapLimit ?? 300;
2072
+ const spanLimit = this.config.mergeMaxSourceSpanMessages ?? 1500;
2073
+ const withPos = [];
2074
+ for (const s of unmerged) {
2075
+ const first = messageOrder.get(s.sourceRange.first);
2076
+ const last = messageOrder.get(s.sourceRange.last);
2077
+ if (first === undefined || last === undefined)
2078
+ continue;
2079
+ if (Math.abs(last - first) > spanLimit)
2080
+ continue; // wide-span quarantine
2081
+ withPos.push({ s, first: Math.min(first, last), last: Math.max(first, last) });
2082
+ }
2083
+ withPos.sort((a, b) => a.first - b.first);
2084
+ let run = [];
2085
+ let runEnd = -Infinity;
2086
+ for (const x of withPos) {
2087
+ if (run.length > 0 && x.first - runEnd > gapLimit) {
2088
+ if (run.length >= threshold)
2089
+ break; // oldest qualifying run wins
2090
+ run = [];
2091
+ runEnd = -Infinity;
2092
+ }
2093
+ run.push(x);
2094
+ runEnd = Math.max(runEnd, x.last);
2095
+ }
2096
+ if (run.length < threshold)
2097
+ return null;
2098
+ return run.slice(0, threshold).map((x) => x.s);
2099
+ }
1476
2100
  checkMergeThreshold() {
2101
+ phaseChannel.report('merge-threshold'); // liveness-watchdog phase
1477
2102
  if (this.config.speculativeProduction) {
1478
2103
  this.checkMergeThresholdRecursive();
1479
2104
  return;
@@ -1490,20 +2115,20 @@ export class AutobiographicalStrategy {
1490
2115
  }
1491
2116
  // Check L1 → L2
1492
2117
  const unmergedL1 = this.summaries.filter(s => s.level === 1 && !s.mergedInto && !queuedL1.has(s.id));
1493
- if (unmergedL1.length >= threshold) {
1494
- const toMerge = unmergedL1.slice(0, threshold);
2118
+ const l1Run = this.contiguousMergeCandidates(unmergedL1, threshold);
2119
+ if (l1Run) {
1495
2120
  this.enqueueMerge({
1496
2121
  level: 2,
1497
- sourceIds: toMerge.map(s => s.id),
2122
+ sourceIds: l1Run.map(s => s.id),
1498
2123
  });
1499
2124
  }
1500
2125
  // Check L2 → L3
1501
2126
  const unmergedL2 = this.summaries.filter(s => s.level === 2 && !s.mergedInto && !queuedL2.has(s.id));
1502
- if (unmergedL2.length >= threshold) {
1503
- const toMerge = unmergedL2.slice(0, threshold);
2127
+ const l2Run = this.contiguousMergeCandidates(unmergedL2, threshold);
2128
+ if (l2Run) {
1504
2129
  this.enqueueMerge({
1505
2130
  level: 3,
1506
- sourceIds: toMerge.map(s => s.id),
2131
+ sourceIds: l2Run.map(s => s.id),
1507
2132
  });
1508
2133
  }
1509
2134
  }
@@ -1545,11 +2170,11 @@ export class AutobiographicalStrategy {
1545
2170
  for (let level = 1; level <= maxLevel; level++) {
1546
2171
  const queued = queuedSources.get(level) ?? new Set();
1547
2172
  const unmerged = this.summaries.filter(s => s.level === level && !getSummaryParentId(s) && !queued.has(s.id));
1548
- if (unmerged.length >= threshold) {
1549
- const toMerge = unmerged.slice(0, threshold);
2173
+ const run = this.contiguousMergeCandidates(unmerged, threshold);
2174
+ if (run) {
1550
2175
  this.enqueueMerge({
1551
2176
  level: level + 1,
1552
- sourceIds: toMerge.map(s => s.id),
2177
+ sourceIds: run.map(s => s.id),
1553
2178
  });
1554
2179
  }
1555
2180
  }
@@ -1658,7 +2283,9 @@ export class AutobiographicalStrategy {
1658
2283
  // parent — the same rule used everywhere else and now in
1659
2284
  // `compressChunkHierarchical`. The cap below is the defense-in-depth.
1660
2285
  const priorSummariesAll = this.summaries
1661
- .filter((s) => !s.mergedInto)
2286
+ // Skip empty-content summaries (see compressChunkHierarchical): an empty
2287
+ // text block in the merge recall header 400s and stalls compression.
2288
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1662
2289
  .filter((s) => {
1663
2290
  for (const lid of s.sourceIds)
1664
2291
  if (sourceLeafIds.has(lid))
@@ -1809,11 +2436,41 @@ export class AutobiographicalStrategy {
1809
2436
  // (present at the start of llmMessages above) and by the prior
1810
2437
  // recall pairs. Same rationale as compressChunkHierarchical.
1811
2438
  const request = {
1812
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
2439
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
2440
+ // raw history that can carry more inline image bytes than the API's
2441
+ // request cap. Dropping the OLDEST images from the summarizer's view is
2442
+ // acceptable policy here — the summary describes the span, it does not
2443
+ // preserve pixels — and membrane error-logs every exercised shed. All
2444
+ // other callers fail loudly instead (no silent transport mutation).
2445
+ shedOversizeImages: true,
2446
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
2447
+ // message left with no content. An empty-content turn (e.g. a silent/skip
2448
+ // turn that produced no text) otherwise reaches the API as an empty text
2449
+ // block → 400 "text content blocks must be non-empty", which throws in the
2450
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
2451
+ // recall-header guard — together they cover every source of the 400.)
2452
+ messages: cleaned
2453
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
2454
+ .filter(m => m.content.length > 0),
1813
2455
  config: {
1814
2456
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1815
- maxTokens: Math.round(targetTokens * 1.5),
2457
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
2458
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
2459
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
2460
+ // off rich memories (stop=max_tokens).
2461
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
1816
2462
  },
2463
+ // Declare the agent's live tools. A summarizer request that replays
2464
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
2465
+ // safety classifier as a foreign agent trace being duplicated ->
2466
+ // deterministic reasoning_extraction refusal of every memory-write
2467
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
2468
+ // tools the live instance runs with is also strictly MORE faithful to
2469
+ // the original context, so it is the KV-honest choice, not a
2470
+ // workaround. Undefined before the first activation of a session --
2471
+ // acceptable: those chunks stay raw and are retried after the agent's
2472
+ // first turn (see the defer guard in compressChunkHierarchical).
2473
+ tools: ctx.tools,
1817
2474
  };
1818
2475
  const callStart = Date.now();
1819
2476
  let logResponse;
@@ -1827,6 +2484,13 @@ export class AutobiographicalStrategy {
1827
2484
  .map(b => b.text)
1828
2485
  .join('\n');
1829
2486
  logResponse = mergedText;
2487
+ // Empty merged generation: skip the merge entirely (do NOT push or mark
2488
+ // sources merged) so we never store/recall an empty summary. Sources stay
2489
+ // unmerged and can be retried.
2490
+ if (!mergedText.trim()) {
2491
+ console.warn(`[autobiographical] empty merged L${targetLevel} summary (${sources.length} sources) — skipping merge`);
2492
+ return;
2493
+ }
1830
2494
  // Compute source range from constituent summaries
1831
2495
  const sourceRange = {
1832
2496
  first: sources[0].sourceRange.first,
@@ -1837,7 +2501,14 @@ export class AutobiographicalStrategy {
1837
2501
  id: `L${targetLevel}-${this.nextSummaryIdCounter()}`,
1838
2502
  level: targetLevel,
1839
2503
  content: mergedText,
1840
- tokens: Math.ceil(mergedText.length / 4),
2504
+ // Exact when available — see the L1 site.
2505
+ tokens: response.usage?.outputTokens &&
2506
+ response.usage.outputTokens > 0 &&
2507
+ // outputTokens includes scratch thinking when present — only exact
2508
+ // when the whole response is the summary text itself.
2509
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
2510
+ ? response.usage.outputTokens
2511
+ : Math.ceil(mergedText.length / 3),
1841
2512
  sourceLevel,
1842
2513
  sourceIds,
1843
2514
  sourceRange,
@@ -1901,11 +2572,20 @@ export class AutobiographicalStrategy {
1901
2572
  * See `docs/adaptive-resolution-design.md` §3, §5.
1902
2573
  */
1903
2574
  selectAdaptive(store, budget) {
2575
+ phaseChannel.report('context-build'); // liveness-watchdog phase
1904
2576
  this.rsBegin();
1905
2577
  const entries = [];
1906
2578
  const maxTokens = budget.maxTokens - budget.reserveForResponse;
2579
+ const overBudgetGraceRatio = Math.max(0, this.config.overBudgetGraceRatio ?? 0);
2580
+ const rejectionBudget = Math.floor(maxTokens * (1 + overBudgetGraceRatio));
2581
+ // Closed-loop calibration: apply the persisted multiplier BEFORE any
2582
+ // estimate is taken this compile.
2583
+ this.loadCalibration(store);
1907
2584
  const messages = store.getAll();
1908
2585
  const msgCap = this.config.maxMessageTokens;
2586
+ // Post-strip estimates (see postStripEstimates): every budgeting site in
2587
+ // this method prices a message the way the stripped render will cost it.
2588
+ const pse = this.postStripEstimates(store);
1909
2589
  // ----- 1. Build head/tail sets and emit head entries -----
1910
2590
  const headStart = this.getHeadWindowStartIndex(store);
1911
2591
  const headEnd = this.getHeadWindowEnd(store);
@@ -1919,8 +2599,8 @@ export class AutobiographicalStrategy {
1919
2599
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
1920
2600
  const msg = messages[i];
1921
2601
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
1922
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
1923
- if (totalTokens + tokens > maxTokens)
2602
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2603
+ if (totalTokens + tokens > rejectionBudget)
1924
2604
  break;
1925
2605
  entries.push({
1926
2606
  index: entries.length,
@@ -1941,7 +2621,7 @@ export class AutobiographicalStrategy {
1941
2621
  const effectiveRecentStart = Math.max(recentStart, headEnd);
1942
2622
  for (let i = effectiveRecentStart; i < messages.length; i++) {
1943
2623
  const msg = messages[i];
1944
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2624
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
1945
2625
  tailMessageIds.add(msg.id);
1946
2626
  tailTokens += tokens;
1947
2627
  }
@@ -1958,6 +2638,11 @@ export class AutobiographicalStrategy {
1958
2638
  // Pinned-position set so the picker doesn't fold messages the user
1959
2639
  // explicitly marked as keep-raw. Built once and reused.
1960
2640
  const pinnedSet = this.pinnedPositions(messages);
2641
+ // V2 dynamic-pin fold-depth bounds (level / maxLevel). A position with a
2642
+ // bound is NOT a classic force-raw pin — the KV-stable controller must be
2643
+ // able to move it to/within its bound — so it renders as `pinned: false`
2644
+ // carrying `pinLevel` / `pinMaxLevel` instead.
2645
+ const pinBounds = this.pinLevelBounds(messages);
1961
2646
  // O(1) summary lookup for findAncestorAt — avoids O(summaries) find()
1962
2647
  // calls during emission.
1963
2648
  const summariesById = new Map();
@@ -1968,16 +2653,22 @@ export class AutobiographicalStrategy {
1968
2653
  const msg = messages[i];
1969
2654
  const ch = chunksByMessageId.get(msg.id);
1970
2655
  const tokens = msgCap > 0
1971
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1972
- : store.estimateTokens(msg);
2656
+ ? Math.min(pse[i], msgCap + 50)
2657
+ : pse[i];
2658
+ const bound = pinBounds.get(i);
1973
2659
  pickerChunks.push({
1974
2660
  id: msg.id,
1975
2661
  sequence: i,
1976
2662
  rawTokens: tokens,
1977
2663
  currentResolution: this.resolutions.get(msg.id) ?? 0,
1978
2664
  lockedByAgent: this.locked.has(msg.id),
1979
- pinned: pinnedSet.has(i),
2665
+ // A classic pin (in pinnedSet with no level bound) stays force-raw. A
2666
+ // leveled pin is not force-raw; it carries its bound instead.
2667
+ pinned: pinnedSet.has(i) && bound === undefined,
2668
+ pinLevel: bound?.level,
2669
+ pinMaxLevel: bound?.maxLevel,
1980
2670
  l1Id: ch?.summaryId,
2671
+ salience: AutobiographicalStrategy.staticSalience(msg),
1981
2672
  });
1982
2673
  }
1983
2674
  // Also include head and tail in PickerChunks (so token accounting matches)
@@ -1985,8 +2676,8 @@ export class AutobiographicalStrategy {
1985
2676
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
1986
2677
  const msg = messages[i];
1987
2678
  const tokens = msgCap > 0
1988
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1989
- : store.estimateTokens(msg);
2679
+ ? Math.min(pse[i], msgCap + 50)
2680
+ : pse[i];
1990
2681
  pickerChunks.push({
1991
2682
  id: msg.id,
1992
2683
  sequence: i,
@@ -2000,8 +2691,8 @@ export class AutobiographicalStrategy {
2000
2691
  for (let i = effectiveRecentStart; i < messages.length; i++) {
2001
2692
  const msg = messages[i];
2002
2693
  const tokens = msgCap > 0
2003
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2004
- : store.estimateTokens(msg);
2694
+ ? Math.min(pse[i], msgCap + 50)
2695
+ : pse[i];
2005
2696
  pickerChunks.push({
2006
2697
  id: msg.id,
2007
2698
  sequence: i,
@@ -2022,7 +2713,13 @@ export class AutobiographicalStrategy {
2022
2713
  recallPairTokens.set(s.id, s.tokens + 20);
2023
2714
  }
2024
2715
  // ----- 4. Run the picker -----
2025
- const totalBudget = maxTokens - totalTokens; // tokens left after head
2716
+ // The picker's token count ALREADY includes the pinned head+tail (it gets
2717
+ // headTokens/tailTokens in pickerInputs and result.finalTokens covers them).
2718
+ // So the budget it folds against is the full maxTokens — NOT maxTokens-head,
2719
+ // which double-counts the head (reserves it twice: once here, once because
2720
+ // finalTokens already includes it). The old form threw ~head-tokens early at
2721
+ // tight budgets and quietly under-used the budget by ~head everywhere.
2722
+ const totalBudget = maxTokens;
2026
2723
  const slack = this.config.compressionSlackRatio ?? 0.1;
2027
2724
  const foldingBudget = {
2028
2725
  totalBudget,
@@ -2042,6 +2739,14 @@ export class AutobiographicalStrategy {
2042
2739
  };
2043
2740
  const picker = this.buildPicker(pickerInputs);
2044
2741
  const result = picker.run(pickerInputs, foldingBudget);
2742
+ // Every trust-region override is loud (design §13.4) — silence was half
2743
+ // of the 2026-07-12 incident.
2744
+ const plan = this._lastKvStable?.lastPlan();
2745
+ if (plan?.override) {
2746
+ console.error(`[kv-escalation] override=${plan.override} perturbation=${plan.perturbation}` +
2747
+ ` tokens=${plan.tokens} budget=${foldingBudget.totalBudget}` +
2748
+ ` (see adaptive-resolution-design.md §13.4)`);
2749
+ }
2045
2750
  // Commit the new resolutions back to strategy state for next compile.
2046
2751
  // Persist to chronicle only if anything actually changed — avoids
2047
2752
  // unnecessary state-slot writes on no-op compiles (which is the common
@@ -2083,9 +2788,9 @@ export class AutobiographicalStrategy {
2083
2788
  // an OverBudgetError to the host rather than silently dropping entries.
2084
2789
  // The strategy has done all it can; the application has to decide what
2085
2790
  // to do next (raise budget, switch model, drop windows, etc.).
2086
- if (result.exhausted && result.finalTokens > totalBudget) {
2791
+ if (result.exhausted && result.finalTokens > rejectionBudget) {
2087
2792
  throw new OverBudgetError({
2088
- budget: totalBudget,
2793
+ budget: rejectionBudget,
2089
2794
  actual: result.finalTokens,
2090
2795
  diagnostics: {
2091
2796
  headTokens,
@@ -2136,7 +2841,7 @@ export class AutobiographicalStrategy {
2136
2841
  // (`maxMessageTokens` is for per-message caps on chat / tool
2137
2842
  // results, not for sharded bodyGroup composites.)
2138
2843
  const tokens = this.estimateTokens(content);
2139
- if (totalTokens + tokens > maxTokens) {
2844
+ if (totalTokens + tokens > rejectionBudget) {
2140
2845
  currentRun = null;
2141
2846
  return false;
2142
2847
  }
@@ -2169,7 +2874,7 @@ export class AutobiographicalStrategy {
2169
2874
  sourceRelation: 'derived',
2170
2875
  };
2171
2876
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2172
- if (totalTokens + pairTokens > maxTokens) {
2877
+ if (totalTokens + pairTokens > rejectionBudget) {
2173
2878
  currentRun = null;
2174
2879
  return false;
2175
2880
  }
@@ -2239,8 +2944,8 @@ export class AutobiographicalStrategy {
2239
2944
  const resolution = result.finalResolutions.get(msg.id) ?? 0;
2240
2945
  if (resolution === 0) {
2241
2946
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2242
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2243
- if (totalTokens + tokens > maxTokens)
2947
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2948
+ if (totalTokens + tokens > rejectionBudget)
2244
2949
  break;
2245
2950
  entries.push({
2246
2951
  index: entries.length,
@@ -2257,8 +2962,8 @@ export class AutobiographicalStrategy {
2257
2962
  const ancestor = this.findAncestorAt(msg.id, resolution, chunksByMessageId, summariesById);
2258
2963
  if (!ancestor) {
2259
2964
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2260
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2261
- if (totalTokens + tokens > maxTokens)
2965
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2966
+ if (totalTokens + tokens > rejectionBudget)
2262
2967
  break;
2263
2968
  entries.push({
2264
2969
  index: entries.length,
@@ -2291,7 +2996,7 @@ export class AutobiographicalStrategy {
2291
2996
  sourceRelation: 'derived',
2292
2997
  };
2293
2998
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2294
- if (totalTokens + pairTokens > maxTokens)
2999
+ if (totalTokens + pairTokens > rejectionBudget)
2295
3000
  break;
2296
3001
  entries.push(questionEntry);
2297
3002
  entries.push(answerEntry);
@@ -2301,7 +3006,7 @@ export class AutobiographicalStrategy {
2301
3006
  }
2302
3007
  }
2303
3008
  // ----- 6. Emit tail entries newest-first eviction (matches existing behavior) -----
2304
- const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
3009
+ const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, rejectionBudget, totalTokens, pse);
2305
3010
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
2306
3011
  // ----- 7. Post-process: merge consecutive raw entries from the same bodyGroup -----
2307
3012
  // Both head and tail emission paths emit shards as separate ContextEntries.
@@ -2312,6 +3017,13 @@ export class AutobiographicalStrategy {
2312
3017
  const merged = this.mergeAdjacentBodyGroupRaw(entries, store);
2313
3018
  this.pruneToolEntries(merged);
2314
3019
  this.trimOrphanedToolUse(merged);
3020
+ // Full pairing invariant over the final rendered context — catches the
3021
+ // mid-list orphans the trailing/leading trims can't (bug 6.7). The adaptive
3022
+ // path has the same mid-list orphan producers as hierarchical (budget
3023
+ // `break`s between pair members, raw emission interleaved with recall
3024
+ // pairs), and FKM defaults autobiographical strategies onto this path — so
3025
+ // the guard has to run here too. It's a no-op on already-valid output.
3026
+ this.enforceToolPairing(merged);
2315
3027
  // Strip stale images BEFORE placing markers and committing stats, so both
2316
3028
  // describe the post-strip context the agent actually receives.
2317
3029
  this.applyImageStripping(merged, store);
@@ -2322,8 +3034,91 @@ export class AutobiographicalStrategy {
2322
3034
  // placement is the dominant KV lever).
2323
3035
  this.placeCacheMarkers(merged, headMessageIds, tailMessageIds);
2324
3036
  this.rsEnd();
3037
+ // Closed-loop calibration bookkeeping: the committed render stats total
3038
+ // (in CURRENT calibrated units) is what this compile claims the request
3039
+ // will cost — reportRealInputTokens compares provider usage against it.
3040
+ this._storeView = store;
3041
+ const rs = this.getRenderStats(store);
3042
+ this._lastCompileEstimate =
3043
+ rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens +
3044
+ rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens;
3045
+ this._calibrationArmed = true; // exactly one sample per compile
2325
3046
  return merged;
2326
3047
  }
3048
+ _lastCompileEstimate = 0;
3049
+ _storeView = null;
3050
+ /**
3051
+ * Closed-loop estimator calibration (2026-07-12). Feed the REAL input total
3052
+ * for a request built from the latest compile (fresh + cache_read +
3053
+ * cache_creation, minus non-window overhead the caller knows about, e.g.
3054
+ * tool schemas). Maintains an EMA of real/estimated and applies it as the
3055
+ * store's global multiplier, persisted across restarts. The per-class rates
3056
+ * carry the shape; this carries the residual level.
3057
+ */
3058
+ reportRealInputTokens(realTotal) {
3059
+ if (!Number.isFinite(realTotal) || realTotal <= 0)
3060
+ return;
3061
+ const est = this._lastCompileEstimate;
3062
+ if (!est || est <= 0)
3063
+ return;
3064
+ // ARM-ONCE-PER-COMPILE (2026-07-12 regression fix). A turn makes MANY API
3065
+ // calls — tool-use rounds and max_tokens continuations each append to the
3066
+ // request — so only the FIRST completion after a compile was built from
3067
+ // the window this estimate describes. Feeding later calls compares a grown
3068
+ // request against the original estimate: ratios of 2.0-2.3 (est=224k
3069
+ // real=504k) drove the multiplier 1.0 -> 2.37 in minutes, inflating every
3070
+ // estimate (the fold floor went 62k -> 108k on unchanged content) and
3071
+ // exhausting the picker on every wake. One sample per compile, always.
3072
+ if (!this._calibrationArmed)
3073
+ return;
3074
+ this._calibrationArmed = false;
3075
+ const ratio = realTotal / est;
3076
+ // SANITY BAND: a representative sample sits near 1. Anything wilder is a
3077
+ // structural mismatch (a request we didn't compile, a partial compile, a
3078
+ // provider quirk) — never evidence about chars-per-token. Log, don't learn.
3079
+ if (ratio < 0.6 || ratio > 1.8) {
3080
+ console.error(`[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ` +
3081
+ `(est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k) — ` +
3082
+ `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}`);
3083
+ return;
3084
+ }
3085
+ const current = this._calibration;
3086
+ const observed = ratio * current; // back out the multiplier already applied
3087
+ const alpha = 0.2; // slow EMA: one wild request shouldn't yank the ruler
3088
+ const next = current + alpha * (observed - current);
3089
+ const clamped = Math.min(1.8, Math.max(0.6, next));
3090
+ if (Math.abs(clamped - current) / current > 0.02) {
3091
+ console.error(`[estimator-calibration] real/est=${ratio.toFixed(2)} ` +
3092
+ `multiplier ${current.toFixed(2)} -> ${clamped.toFixed(2)} (est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k)`);
3093
+ }
3094
+ this._calibration = clamped;
3095
+ this.applyCalibration();
3096
+ try {
3097
+ this.store?.setStateJson(this.calibrationStateId, { multiplier: this._calibration, at: Date.now() });
3098
+ }
3099
+ catch { /* persistence is best-effort */ }
3100
+ }
3101
+ _calibrationArmed = false;
3102
+ _calibration = 1;
3103
+ _calibrationLoaded = false;
3104
+ applyCalibration() {
3105
+ this._storeView?.setTokenCalibration?.(this._calibration);
3106
+ }
3107
+ /** Load the persisted multiplier once and push it into the store view. */
3108
+ loadCalibration(store) {
3109
+ this._storeView = store;
3110
+ if (!this._calibrationLoaded) {
3111
+ this._calibrationLoaded = true;
3112
+ try {
3113
+ const saved = this.store?.getStateJson(this.calibrationStateId);
3114
+ if (saved && Number.isFinite(saved.multiplier)) {
3115
+ this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier));
3116
+ }
3117
+ }
3118
+ catch { /* absent slot is fine */ }
3119
+ }
3120
+ this.applyCalibration();
3121
+ }
2327
3122
  /**
2328
3123
  * Place up to four `cache_control` breakpoints across the final ordered
2329
3124
  * entries: the head/system boundary, the end of the folded history (the
@@ -2378,13 +3173,23 @@ export class AutobiographicalStrategy {
2378
3173
  mergeAdjacentBodyGroupRaw(entries, store) {
2379
3174
  if (entries.length === 0)
2380
3175
  return entries;
2381
- // Look up bodyGroupId by sourceMessageId via the message store.
2382
- const groupOf = (sourceMessageId) => {
3176
+ // Look up bodyGroup metadata by sourceMessageId, memoized for the
3177
+ // duration of this pass. Entries reference the same messages repeatedly
3178
+ // (run-extension checks + the shardIndex sort comparator below), and
3179
+ // store.get() also resolves blobs — fetch each message at most once.
3180
+ const shardMeta = new Map();
3181
+ const metaOf = (sourceMessageId) => {
2383
3182
  if (!sourceMessageId)
2384
3183
  return undefined;
2385
- const m = store.get(sourceMessageId);
2386
- return m?.bodyGroupId;
3184
+ let meta = shardMeta.get(sourceMessageId);
3185
+ if (!meta) {
3186
+ const m = store.get(sourceMessageId);
3187
+ meta = { groupId: m?.bodyGroupId, shardIndex: m?.shardIndex };
3188
+ shardMeta.set(sourceMessageId, meta);
3189
+ }
3190
+ return meta;
2387
3191
  };
3192
+ const groupOf = (sourceMessageId) => metaOf(sourceMessageId)?.groupId;
2388
3193
  const out = [];
2389
3194
  let i = 0;
2390
3195
  while (i < entries.length) {
@@ -2413,9 +3218,9 @@ export class AutobiographicalStrategy {
2413
3218
  // ordering. (Head/tail emission keeps chronological order, but defending
2414
3219
  // against reorderings is cheap.)
2415
3220
  const sortedRun = [...run].sort((a, b) => {
2416
- const ma = a.sourceMessageId ? store.get(a.sourceMessageId) : null;
2417
- const mb = b.sourceMessageId ? store.get(b.sourceMessageId) : null;
2418
- return (ma?.shardIndex ?? 0) - (mb?.shardIndex ?? 0);
3221
+ const sa = metaOf(a.sourceMessageId)?.shardIndex ?? 0;
3222
+ const sb = metaOf(b.sourceMessageId)?.shardIndex ?? 0;
3223
+ return sa - sb;
2419
3224
  });
2420
3225
  // Build merged text content. Non-text blocks (rare in shards) are
2421
3226
  // preserved on the first shard's entry only.
@@ -2461,13 +3266,83 @@ export class AutobiographicalStrategy {
2461
3266
  */
2462
3267
  buildPicker(inputs) {
2463
3268
  if (this.config.foldingStrategy === 'kv-stable') {
2464
- return new Picker(new KvStableStrategy(inputs, {
3269
+ const strategy = new KvStableStrategy(inputs, {
2465
3270
  reachTokens: this.config.kvStableReachTokens,
3271
+ qualityGapRatio: this.config.kvStableQualityGapRatio,
2466
3272
  mergeThreshold: this.config.mergeThreshold,
2467
- }));
3273
+ });
3274
+ this._lastKvStable = strategy;
3275
+ return new Picker(strategy);
2468
3276
  }
3277
+ this._lastKvStable = null;
2469
3278
  return this.getAdaptivePicker();
2470
3279
  }
3280
+ /** The kv-stable strategy instance behind the most recent compile — kept for
3281
+ * `[kv-escalation]` observability (design §13.4: every override is loud). */
3282
+ _lastKvStable = null;
3283
+ /**
3284
+ * Static salience prior (design §13.3) — "is the window the only copy?".
3285
+ * Content whose payload is externalized folds cheap: tool blocks
3286
+ * (re-derivable), fenced code (usually written to disk), images (the file/
3287
+ * CDN keeps them), bare link drops. Conversation exists nowhere but the
3288
+ * chronicle, so it stays at 1. Returns a value in [0.2, 1]; cheap,
3289
+ * deterministic, computed per message at picker-input construction.
3290
+ */
3291
+ static staticSalience(msg) {
3292
+ let totalChars = 0;
3293
+ let externalChars = 0;
3294
+ for (const block of msg.content) {
3295
+ const b = block;
3296
+ switch (b.type) {
3297
+ case 'text': {
3298
+ const t = b.text ?? '';
3299
+ totalChars += t.length;
3300
+ // Fenced code blocks.
3301
+ const fences = t.split('```');
3302
+ for (let i = 1; i < fences.length; i += 2)
3303
+ externalChars += fences[i].length;
3304
+ // Bare link-drop lines (the URL is the payload).
3305
+ for (const line of t.split('\n')) {
3306
+ const trimmed = line.trim();
3307
+ if (/^https?:\/\/\S+$/.test(trimmed))
3308
+ externalChars += trimmed.length;
3309
+ }
3310
+ break;
3311
+ }
3312
+ case 'tool_use': {
3313
+ const n = JSON.stringify(b.input ?? {}).length;
3314
+ totalChars += n;
3315
+ externalChars += n;
3316
+ break;
3317
+ }
3318
+ case 'tool_result': {
3319
+ const n = typeof b.content === 'string'
3320
+ ? b.content.length
3321
+ : JSON.stringify(b.content ?? '').length;
3322
+ totalChars += n;
3323
+ externalChars += n;
3324
+ break;
3325
+ }
3326
+ case 'image': {
3327
+ // Estimate parity with the renderer's flat image cost; the payload
3328
+ // lives in the file/CDN, so it is fully externalized.
3329
+ totalChars += 6400; // ≈1600 tokens × 4 chars
3330
+ externalChars += 6400;
3331
+ break;
3332
+ }
3333
+ default: {
3334
+ const t = b.text ?? '';
3335
+ totalChars += t.length;
3336
+ }
3337
+ }
3338
+ }
3339
+ if (totalChars <= 0)
3340
+ return 1;
3341
+ const externalized = Math.min(1, externalChars / totalChars);
3342
+ // Fully-externalized content bottoms out at 0.2 — cheap, never free
3343
+ // (hard protections, not salience, are what make content unfoldable).
3344
+ return Math.max(0.2, 1 - 0.8 * externalized);
3345
+ }
2471
3346
  /**
2472
3347
  * Walk the summary tree to find the L_k ancestor of a message.
2473
3348
  * Returns null if no ancestor exists at that level (e.g., L_k not yet produced).
@@ -2541,6 +3416,7 @@ export class AutobiographicalStrategy {
2541
3416
  * Matches moltbot's budget waterfall: L3 → L2 → L1 with unused budget flowing down.
2542
3417
  */
2543
3418
  selectHierarchical(store, budget) {
3419
+ phaseChannel.report('context-build'); // liveness-watchdog phase
2544
3420
  this.rsBegin();
2545
3421
  const entries = [];
2546
3422
  const maxTokens = budget.maxTokens - budget.reserveForResponse;
@@ -2616,6 +3492,85 @@ export class AutobiographicalStrategy {
2616
3492
  const { selected: l1Selected, tokensUsed: l1Used } = this.selectL1Summaries(shownL1, l1Effective, l1Remaining);
2617
3493
  selectedSummaries.push(...l1Selected);
2618
3494
  totalSummaryTokens += l1Used;
3495
+ // Phase 3b: coverage repair (bug 6.9). getAntiRedundantSummaries excluded
3496
+ // an L2 (or L3) when ALL of its children were in the CANDIDATE shown-set —
3497
+ // computed before budget selection. If the budget then dropped some of
3498
+ // those children (e.g. KnowledgeStrategy's research L1 cap), the covered
3499
+ // history appears at NEITHER level: a silent memory hole. Re-include any
3500
+ // excluded L2/L3 whose children did not all make the final selection.
3501
+ // Some overlap with the children that DID survive is accepted — coverage
3502
+ // beats perfect dedup here.
3503
+ //
3504
+ // Repairs are additionally bounded by a per-level ALLOWANCE (a fraction of
3505
+ // that level's budget), not just the overall context budget. The
3506
+ // excluded-with-partially-dropped-children state only arises from a store
3507
+ // damaged mid-merge (the crash window at compressChunkHierarchical, or the
3508
+ // legacy setMergedInto index-desync). On such a store MANY L2s can be in
3509
+ // this state at once; without a cap, re-including all of them at full size
3510
+ // would starve the recent window via Phase 4's newest-first eviction. When
3511
+ // repairs exceed the allowance we stop re-including and warn — a corrupted
3512
+ // store announces itself instead of silently trading recent messages for
3513
+ // redundant summaries.
3514
+ {
3515
+ // Allowance = a fraction of the level budget, with a floor tied to the
3516
+ // overall budget so a strategy that zeroes a level budget (e.g.
3517
+ // KnowledgeStrategy prioritising L1) can still repair a handful of
3518
+ // covering summaries, while a corrupted store with dozens of them stays
3519
+ // bounded well short of the recent window.
3520
+ const REPAIR_ALLOWANCE_FRACTION = 0.25;
3521
+ const REPAIR_FLOOR_FRACTION = 0.05;
3522
+ const repairFloor = maxTokens * REPAIR_FLOOR_FRACTION;
3523
+ const l2RepairAllowance = Math.max(l2Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
3524
+ const l3RepairAllowance = Math.max(l3Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
3525
+ const selectedIds = new Set(selectedSummaries.map(s => s.id));
3526
+ const shownL2Ids = new Set(shownL2.map(s => s.id));
3527
+ const shownL3Ids = new Set(shownL3.map(s => s.id));
3528
+ let l2RepairTokens = 0;
3529
+ let l3RepairTokens = 0;
3530
+ let l2RepairsSkipped = 0;
3531
+ let l3RepairsSkipped = 0;
3532
+ // L2s excluded by anti-redundancy: unmerged, not in shownL2.
3533
+ for (const s of this.summaries) {
3534
+ if (s.level !== 2 || s.mergedInto || shownL2Ids.has(s.id))
3535
+ continue;
3536
+ if (s.sourceIds.every(id => selectedIds.has(id)))
3537
+ continue; // truly redundant
3538
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
3539
+ continue;
3540
+ if (l2RepairTokens + s.tokens > l2RepairAllowance) {
3541
+ l2RepairsSkipped++;
3542
+ continue;
3543
+ }
3544
+ selectedSummaries.push(s);
3545
+ totalSummaryTokens += s.tokens;
3546
+ l2RepairTokens += s.tokens;
3547
+ selectedIds.add(s.id);
3548
+ }
3549
+ // L3s excluded by anti-redundancy — after L2 repair, so a repaired L2
3550
+ // counts as selected coverage for its parent L3.
3551
+ for (const s of this.summaries) {
3552
+ if (s.level !== 3 || s.mergedInto || shownL3Ids.has(s.id))
3553
+ continue;
3554
+ if (s.sourceIds.every(id => selectedIds.has(id)))
3555
+ continue;
3556
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
3557
+ continue;
3558
+ if (l3RepairTokens + s.tokens > l3RepairAllowance) {
3559
+ l3RepairsSkipped++;
3560
+ continue;
3561
+ }
3562
+ selectedSummaries.push(s);
3563
+ totalSummaryTokens += s.tokens;
3564
+ l3RepairTokens += s.tokens;
3565
+ selectedIds.add(s.id);
3566
+ }
3567
+ if (l2RepairsSkipped > 0 || l3RepairsSkipped > 0) {
3568
+ console.warn(`[AutobiographicalStrategy] coverage-repair allowance exceeded — ` +
3569
+ `skipped ${l2RepairsSkipped} L2 and ${l3RepairsSkipped} L3 re-inclusions ` +
3570
+ `(store likely corrupted mid-merge). Some covered history may render at ` +
3571
+ `no summary level this pass.`);
3572
+ }
3573
+ }
2619
3574
  // Emit summaries + pinned messages between head and recent windows.
2620
3575
  //
2621
3576
  // Default (positionedRecallPairs=true): one Q/A pair per summary,
@@ -2693,6 +3648,11 @@ export class AutobiographicalStrategy {
2693
3648
  for (const item of items) {
2694
3649
  if (item.kind === 'summary') {
2695
3650
  const summary = item.summary;
3651
+ // Defensive: never emit a recall pair for an empty/bugged summary — an
3652
+ // empty assistant text block triggers a 400. (Production guards too,
3653
+ // but a legacy empty summary may already exist in the store.)
3654
+ if (!summary.content || !summary.content.trim())
3655
+ continue;
2696
3656
  const headerText = this.buildRecallHeader(summary);
2697
3657
  const questionEntry = {
2698
3658
  index: entries.length,
@@ -2797,6 +3757,9 @@ export class AutobiographicalStrategy {
2797
3757
  const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
2798
3758
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
2799
3759
  this.trimOrphanedToolUse(entries);
3760
+ // Full pairing invariant over the final rendered context — catches the
3761
+ // mid-list orphans the trailing/leading trims can't (bug 6.7).
3762
+ this.enforceToolPairing(entries);
2800
3763
  this.pruneToolEntries(entries);
2801
3764
  // Strip stale images before committing stats so RenderStats.total reflects
2802
3765
  // the post-strip context (this path places no cache markers).
@@ -3070,24 +4033,81 @@ export class AutobiographicalStrategy {
3070
4033
  return out;
3071
4034
  }
3072
4035
  /**
3073
- * Rebuild chunk boundaries based on current messages.
4036
+ * Rebuild the chunk list: persisted records own the past; the running-sum
4037
+ * chunker only extends at the frontier, and a chunk is only ever created
4038
+ * once it CLOSES (reaches targetChunkTokens). The trailing partial chunk
4039
+ * is never created and never compressed — eager partial-tail compression
4040
+ * minted a new near-duplicate L1 per rebuild while the tail grew (the
4041
+ * prefix-generation families found fleet-wide in the 2026-07 audit).
3074
4042
  */
3075
4043
  rebuildChunks(store) {
3076
- const messagesToChunk = this.getCompressibleMessages(store);
3077
- // Preserve existing compressed chunks (legacy) and summary linkage (hierarchical)
3078
- const existingCompressed = new Map();
4044
+ this.chunks = [];
4045
+ this.compressionQueue = [];
4046
+ // ---- 1. Materialize persisted records (they OWN their messages). ----
4047
+ const byId = new Map();
4048
+ for (const m of store.getAll())
4049
+ byId.set(m.id, m);
4050
+ const consumed = new Set();
4051
+ let orphaned = 0;
4052
+ for (const rec of this.chunkRecords) {
4053
+ const msgs = [];
4054
+ for (const id of rec.sourceIds) {
4055
+ const m = byId.get(id);
4056
+ if (m)
4057
+ msgs.push(m);
4058
+ }
4059
+ if (msgs.length === 0) {
4060
+ orphaned++;
4061
+ continue;
4062
+ }
4063
+ for (const m of msgs)
4064
+ consumed.add(m.id);
4065
+ const chunk = {
4066
+ index: this.chunks.length,
4067
+ startIndex: -1, // record-derived; filtered-array indices are not meaningful
4068
+ endIndex: -1,
4069
+ messages: msgs,
4070
+ tokens: msgs.reduce((sum, m) => sum + (this.config.attachmentsIgnoreSize
4071
+ ? this.estimateTextOnlyTokens(m)
4072
+ : store.estimateTokens(m)), 0),
4073
+ compressed: rec.compressed,
4074
+ summaryId: rec.summaryId,
4075
+ phaseType: rec.phaseType,
4076
+ recordId: rec.id,
4077
+ };
4078
+ this.chunks.push(chunk);
4079
+ }
4080
+ // ---- 2. Fail closed on the chain-break signature. ----
4081
+ // Most records resolving to zero live messages means message identity
4082
+ // has been rebuilt/renumbered underneath us. Chunking "fresh" ground now
4083
+ // would re-compress already-lived history into duplicate memories.
4084
+ // Halt ALL compression until an operator reconciles the store.
4085
+ if (this.chunkPersistenceEnabled && this.chunkRecords.length >= 3 &&
4086
+ orphaned / this.chunkRecords.length > 0.5) {
4087
+ this.chunkRecordsOrphaned = true;
4088
+ this.compressionQueue = [];
4089
+ if (!this._orphanWarned) {
4090
+ this._orphanWarned = true;
4091
+ console.error(`[autobiographical] FAIL-CLOSED: ${orphaned}/${this.chunkRecords.length} chunk ` +
4092
+ `records resolve to zero live messages (messages chain break / store ` +
4093
+ `reconciliation signature). Compression halted to prevent duplicate ` +
4094
+ `memory formation — reconcile the store before resuming.`);
4095
+ }
4096
+ return;
4097
+ }
4098
+ this.chunkRecordsOrphaned = false;
4099
+ // Queue uncompressed record-backed chunks (crash-recovery: record was
4100
+ // appended but the process died before its L1 landed).
3079
4101
  for (const chunk of this.chunks) {
3080
- if (chunk.compressed) {
3081
- const key = this.chunkKey(chunk);
3082
- existingCompressed.set(key, chunk);
4102
+ if (!chunk.compressed && !(chunk.recordId && this._overlapBlocked.has(chunk.recordId))) {
4103
+ this.compressionQueue.push(chunk.index);
3083
4104
  }
3084
4105
  }
3085
- // Rebuild chunks
3086
- this.chunks = [];
3087
- this.compressionQueue = [];
4106
+ // ---- 3. Chunk the frontier: compressible messages not owned by any record. ----
4107
+ const messagesToChunk = this.getCompressibleMessages(store)
4108
+ .filter(m => !consumed.has(m.id));
3088
4109
  let currentChunk = [];
3089
4110
  let currentTokens = 0;
3090
- // Track start position in the filtered array for chunk boundary metadata
3091
4111
  let chunkFilteredStart = 0;
3092
4112
  for (let i = 0; i < messagesToChunk.length; i++) {
3093
4113
  const msg = messagesToChunk[i];
@@ -3110,22 +4130,53 @@ export class AutobiographicalStrategy {
3110
4130
  continue;
3111
4131
  }
3112
4132
  if (shouldClose) {
3113
- const chunk = this.createChunk(this.chunks.length, chunkFilteredStart, i + 1, currentChunk, currentTokens, existingCompressed);
3114
- this.chunks.push(chunk);
3115
- if (!chunk.compressed) {
3116
- this.compressionQueue.push(chunk.index);
4133
+ const chunk = {
4134
+ index: this.chunks.length,
4135
+ startIndex: chunkFilteredStart,
4136
+ endIndex: i + 1,
4137
+ messages: [...currentChunk],
4138
+ tokens: currentTokens,
4139
+ compressed: false,
4140
+ };
4141
+ // Persist the boundary the moment it closes — from here on this
4142
+ // span is owned and never re-keyed by config drift or restarts.
4143
+ if (this.chunkPersistenceEnabled) {
4144
+ const record = {
4145
+ id: `c-${this.chunkIdCounter++}`,
4146
+ sourceIds: chunk.messages.map(m => m.id),
4147
+ compressed: false,
4148
+ };
4149
+ this.appendChunkRecord(record);
4150
+ chunk.recordId = record.id;
3117
4151
  }
4152
+ this.chunks.push(chunk);
4153
+ this.compressionQueue.push(chunk.index);
3118
4154
  currentChunk = [];
3119
4155
  currentTokens = 0;
3120
4156
  chunkFilteredStart = i + 1;
3121
4157
  }
3122
4158
  }
3123
- if (currentChunk.length >= 4) {
3124
- const chunk = this.createChunk(this.chunks.length, chunkFilteredStart, messagesToChunk.length, currentChunk, currentTokens, existingCompressed);
3125
- this.chunks.push(chunk);
3126
- if (!chunk.compressed) {
3127
- this.compressionQueue.push(chunk.index);
3128
- }
4159
+ // NOTE: no trailing-partial chunk. An unclosed chunk is not a chunk —
4160
+ // it compresses only after the running sum closes it.
4161
+ // ---- 4. L1 holdback: keep the newest X closed chunks out of the
4162
+ // speculative queue (default 1). The chunk at the live edge is the one
4163
+ // most likely to still be in motion (edits, tool-result landings, the
4164
+ // episode it belongs to still resolving); summarize it once a newer chunk
4165
+ // has closed behind it. The queue is rebuilt on every message, so a
4166
+ // held-back chunk is released automatically the moment it ages out of the
4167
+ // window. Demand overrides: a picker `produce` op (enqueueL1ForRange)
4168
+ // marks the chunk demanded, and demanded chunks are never held back —
4169
+ // when folding actually NEEDS the L1, production must not be blocked.
4170
+ const holdback = this.config.l1HoldbackChunks ?? 1;
4171
+ if (holdback > 0 && this.chunks.length > 0) {
4172
+ const cutoff = this.chunks.length - holdback;
4173
+ this.compressionQueue = this.compressionQueue.filter((idx) => {
4174
+ if (idx < cutoff)
4175
+ return true;
4176
+ const ch = this.chunks[idx];
4177
+ const lastId = ch?.messages[ch.messages.length - 1]?.id;
4178
+ return lastId !== undefined && this._demandedL1Chunks.has(lastId);
4179
+ });
3129
4180
  }
3130
4181
  }
3131
4182
  /**
@@ -3206,7 +4257,8 @@ export class AutobiographicalStrategy {
3206
4257
  applyImageStripping(entries, store) {
3207
4258
  const maxLive = this.config.maxLiveImages ?? 0; // 0 = unlimited count
3208
4259
  const depthTokens = this.config.imageStripDepthTokens ?? 0; // 0 = no depth strip
3209
- if (maxLive === 0 && depthTokens === 0)
4260
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4261
+ if (maxLive === 0 && depthTokens === 0 && maxLiveBytes === 0)
3210
4262
  return; // policy disabled
3211
4263
  const messages = store.getAll();
3212
4264
  const posById = new Map();
@@ -3238,6 +4290,7 @@ export class AutobiographicalStrategy {
3238
4290
  .filter(({ idx }) => this.hasImageBlock(entries[idx].content))
3239
4291
  .sort((a, b) => b.pos - a.pos);
3240
4292
  let keptImages = 0;
4293
+ let keptImageBytes = 0;
3241
4294
  for (const { idx, pos } of ordered) {
3242
4295
  const entry = entries[idx];
3243
4296
  const tooDeep = depthTokens > 0 && (pos < 0 || pos < stripStart);
@@ -3245,26 +4298,86 @@ export class AutobiographicalStrategy {
3245
4298
  entry.content = entry.content.map((block) => {
3246
4299
  if (block.type !== 'image')
3247
4300
  return block;
4301
+ const blockBytes = AutobiographicalStrategy.imageBlockBytes(block);
3248
4302
  const overCount = maxLive > 0 && keptImages >= maxLive;
3249
- if (tooDeep || overCount) {
3250
- // The renderer estimates an image at `tokenEstimate ?? 1600` (see
3251
- // message-store/context-log). Hand that back to the bucket, less the
3252
- // placeholder text that replaces it, so the stats match the output.
3253
- const reclaimed = (block.tokenEstimate ?? 1600) - placeholderTokens;
3254
- if (this._rs && reclaimed > 0)
3255
- this._rs[bucket].tokens -= reclaimed;
4303
+ const overBytes = maxLiveBytes > 0 && keptImageBytes + blockBytes > maxLiveBytes;
4304
+ if (tooDeep || overCount || overBytes) {
4305
+ // Stats-neutral (2026-07-12): every budgeting site now tallies at
4306
+ // POST-STRIP prices (see postStripEstimates), so the bucket never
4307
+ // charged this image at full weight — reclaiming here would
4308
+ // double-decrement. The strip pass only swaps the block.
4309
+ void bucket;
4310
+ void placeholderTokens;
3256
4311
  return { type: 'text', text: AutobiographicalStrategy.IMAGE_PLACEHOLDER };
3257
4312
  }
3258
4313
  keptImages++;
4314
+ keptImageBytes += blockBytes;
3259
4315
  return block;
3260
4316
  });
3261
4317
  }
3262
4318
  }
4319
+ /**
4320
+ * Post-strip token estimate per message index (2026-07-12 tail-starvation
4321
+ * fix). Mirrors `applyImageStripping`: an image beyond the `maxLiveImages`
4322
+ * newest (counted newest-first) or deeper than `imageStripDepthTokens` of
4323
+ * raw estimate from the live end renders as a placeholder — so every place
4324
+ * that BUDGETS messages (recent-window walk-back, head/tail sums, middle
4325
+ * chunk sizes) must cost it as one. Pricing stripped images at their full
4326
+ * estimate collapsed an image-dense tail to a fraction of its configured
4327
+ * size (42k rendered of a 120k window), and pricing them post-strip in the
4328
+ * walk-back alone made the picker's raw-priced tail overflow the budget
4329
+ * (318k) — the estimate must be consistent EVERYWHERE.
4330
+ */
4331
+ postStripEstimates(store) {
4332
+ const messages = store.getAll();
4333
+ const out = new Array(messages.length);
4334
+ const stripDepth = this.config.imageStripDepthTokens ?? 0;
4335
+ const maxLive = this.config.maxLiveImages ?? 0;
4336
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4337
+ const stripActive = stripDepth > 0 || maxLive > 0 || maxLiveBytes > 0;
4338
+ const placeholderTokens = Math.ceil(AutobiographicalStrategy.IMAGE_PLACEHOLDER.length / 4);
4339
+ let liveImagesSeen = 0;
4340
+ let liveImageBytes = 0;
4341
+ let rawDepth = 0; // raw-estimate depth from the newest message (mirrors getImageStripStart)
4342
+ for (let i = messages.length - 1; i >= 0; i--) {
4343
+ const raw = store.estimateTokens(messages[i]);
4344
+ let est = raw;
4345
+ if (stripActive) {
4346
+ for (const b of messages[i].content) {
4347
+ if (b.type !== 'image')
4348
+ continue;
4349
+ const bytes = AutobiographicalStrategy.imageBlockBytes(b);
4350
+ const beyondDepth = stripDepth > 0 && rawDepth > stripDepth;
4351
+ const beyondCount = maxLive > 0 && liveImagesSeen >= maxLive;
4352
+ const beyondBytes = maxLiveBytes > 0 && liveImageBytes + bytes > maxLiveBytes;
4353
+ if (beyondDepth || beyondCount || beyondBytes) {
4354
+ const imgEst = b.tokenEstimate ?? 1600;
4355
+ est -= Math.max(0, imgEst - placeholderTokens);
4356
+ }
4357
+ else {
4358
+ liveImagesSeen++;
4359
+ liveImageBytes += bytes;
4360
+ }
4361
+ }
4362
+ }
4363
+ rawDepth += raw;
4364
+ out[i] = est;
4365
+ }
4366
+ return out;
4367
+ }
4368
+ /** Byte wall default: 20MB of base64 (API total-request cap is 32MB). */
4369
+ static DEFAULT_MAX_LIVE_IMAGE_BYTES = 20 * 1024 * 1024;
4370
+ /** Base64 payload size of an image block (0 for non-base64 sources). */
4371
+ static imageBlockBytes(b) {
4372
+ const src = b.source;
4373
+ return typeof src?.data === 'string' ? src.data.length : 0;
4374
+ }
3263
4375
  getRecentWindowStart(store) {
3264
4376
  const messages = store.getAll();
4377
+ const pse = this.postStripEstimates(store);
3265
4378
  let tokens = 0;
3266
4379
  for (let i = messages.length - 1; i >= 0; i--) {
3267
- tokens += store.estimateTokens(messages[i]);
4380
+ tokens += pse[i];
3268
4381
  if (tokens > this.config.recentWindowTokens) {
3269
4382
  let boundary = i + 1;
3270
4383
  // Don't split a tool_use/tool_result pair: if the message at the boundary
@@ -3345,6 +4458,171 @@ export class AutobiographicalStrategy {
3345
4458
  }
3346
4459
  }
3347
4460
  }
4461
+ /** Placeholder body for a stub tool_result inserted by enforceToolPairing. */
4462
+ static STUB_TOOL_RESULT_TEXT = '[tool result unavailable — omitted during context compression]';
4463
+ /**
4464
+ * Final post-selection tool-pairing validator (bug 6.7).
4465
+ *
4466
+ * The Anthropic API requires every `tool_use` block to be answered by a
4467
+ * matching `tool_result` in the immediately-following message, and every
4468
+ * `tool_result` to answer a `tool_use` in the immediately-preceding
4469
+ * message. Selection can violate this mid-list in ways the trailing
4470
+ * (`trimOrphanedToolUse`) and leading orphan trims don't catch:
4471
+ *
4472
+ * - a budget `break` cutting between a raw pin pair's two messages;
4473
+ * - the uncompressed-chunk fallback emitting a raw tool_result whose
4474
+ * tool_use chunk already compressed (or vice versa);
4475
+ * - a recall pair or pin interleaving between a tool_use and its result.
4476
+ *
4477
+ * Repair policy prefers preserving content over dropping:
4478
+ *
4479
+ * - a tool_use whose result is missing from the next entry first triggers
4480
+ * a short look-ahead: if the genuine (displaced) result is a few entries
4481
+ * down it is MOVED up into position (see
4482
+ * {@link relocateOrDropMissingResults}); only when no real result exists
4483
+ * is a STUB tool_result emitted. Either way the result block is merged
4484
+ * into the next entry when that entry already carries results for this
4485
+ * cycle, or inserted as a new user entry;
4486
+ * - a tool_result whose tool_use is not in the immediately-preceding
4487
+ * entry (and was not relocated) is dropped — there is no safe way to
4488
+ * stub a tool_use, so the result's information content survives only if
4489
+ * its use is adjacent; an entry left empty is replaced with a
4490
+ * placeholder text block.
4491
+ *
4492
+ * Runs as a structural pass over the rendered context in BOTH render
4493
+ * paths — `selectHierarchical` (downstream of `selectL1Summaries`, so
4494
+ * subclass overrides like KnowledgeStrategy are covered) and
4495
+ * `selectAdaptive` (the path FKM defaults onto). It's a no-op on
4496
+ * already-valid output.
4497
+ */
4498
+ enforceToolPairing(entries) {
4499
+ let prevUseIds = new Set();
4500
+ for (let i = 0; i < entries.length; i++) {
4501
+ const entry = entries[i];
4502
+ // --- Rule A: every tool_result must answer a tool_use in the
4503
+ // immediately-preceding entry (and only once). Drop orphans/dupes. ---
4504
+ if (entry.content.some(b => b.type === 'tool_result')) {
4505
+ const seen = new Set();
4506
+ const filtered = entry.content.filter(b => {
4507
+ if (b.type !== 'tool_result')
4508
+ return true;
4509
+ if (!prevUseIds.has(b.toolUseId) || seen.has(b.toolUseId))
4510
+ return false;
4511
+ seen.add(b.toolUseId);
4512
+ return true;
4513
+ });
4514
+ if (filtered.length !== entry.content.length) {
4515
+ entry.content = filtered.length > 0
4516
+ ? filtered
4517
+ : [{ type: 'text', text: '[tool call omitted]' }];
4518
+ }
4519
+ }
4520
+ // --- Rule B: every tool_use in the PREVIOUS entry must be answered
4521
+ // by this entry. Stub any that aren't. ---
4522
+ if (prevUseIds.size > 0) {
4523
+ const answered = new Set();
4524
+ for (const b of entry.content) {
4525
+ if (b.type === 'tool_result')
4526
+ answered.add(b.toolUseId);
4527
+ }
4528
+ const missing = [...prevUseIds].filter(id => !answered.has(id));
4529
+ if (missing.length > 0) {
4530
+ // Look-ahead relocation: the genuine result for a "missing" id is
4531
+ // often sitting a few entries down, displaced by an interleaved
4532
+ // recall pair / pin (it will otherwise be dropped as an orphan by
4533
+ // Rule A when we reach it). Move the real block up into position and
4534
+ // only stub the ids for which no real result exists — so tool output
4535
+ // is preserved, not silently replaced by a placeholder.
4536
+ const results = this.relocateOrDropMissingResults(entries, i, missing);
4537
+ if (answered.size > 0) {
4538
+ // This entry already carries results for the cycle — prepend the
4539
+ // relocated/stub results so all results for the preceding tool_use
4540
+ // sit together (the API wants tool_result blocks at the head of
4541
+ // the message).
4542
+ entry.content = [...results, ...entry.content];
4543
+ }
4544
+ else {
4545
+ // Not a results entry at all — insert a synthetic user entry
4546
+ // between the tool_use entry and this one.
4547
+ entries.splice(i, 0, {
4548
+ index: i,
4549
+ participant: 'user',
4550
+ sourceRelation: 'derived',
4551
+ content: results,
4552
+ });
4553
+ // The stub entry (no tool_use blocks) is now at index i; the
4554
+ // current entry moved to i+1 and is re-processed next iteration
4555
+ // with an empty prevUseIds (its orphan results, if any, were
4556
+ // already filtered above and none survived — Rule A matched
4557
+ // against the same prevUseIds and answered.size === 0).
4558
+ prevUseIds = new Set();
4559
+ continue;
4560
+ }
4561
+ }
4562
+ }
4563
+ prevUseIds = new Set();
4564
+ for (const b of entry.content) {
4565
+ if (b.type === 'tool_use')
4566
+ prevUseIds.add(b.id);
4567
+ }
4568
+ }
4569
+ // Tail: an entry that mixes tool_use with other blocks can survive
4570
+ // trimOrphanedToolUse (which only pops pure use-without-result tails).
4571
+ if (prevUseIds.size > 0) {
4572
+ entries.push({
4573
+ index: entries.length,
4574
+ participant: 'user',
4575
+ sourceRelation: 'derived',
4576
+ content: [...prevUseIds].map(id => ({
4577
+ type: 'tool_result',
4578
+ toolUseId: id,
4579
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
4580
+ })),
4581
+ });
4582
+ }
4583
+ // Reindex after any splices/appends.
4584
+ for (let i = 0; i < entries.length; i++)
4585
+ entries[i].index = i;
4586
+ }
4587
+ /**
4588
+ * For each `missing` tool_use id (a use in the preceding entry with no
4589
+ * adjacent result), return the result block to place next to it:
4590
+ *
4591
+ * - if the genuine result exists within a short look-ahead window after
4592
+ * `afterIndex`, MOVE it up into position (removing it from its source
4593
+ * entry — replacing a now-empty entry with a placeholder) so the real
4594
+ * tool output survives the repair;
4595
+ * - otherwise emit a stub.
4596
+ *
4597
+ * tool_use ids are unique, so the single result carrying a given id is
4598
+ * unambiguously the answer to that use — relocating it cannot break any
4599
+ * other pairing. Results for `missing` ids never sit at or before
4600
+ * `afterIndex` (that entry's results are already matched), so the search
4601
+ * starts at `afterIndex + 1`.
4602
+ */
4603
+ relocateOrDropMissingResults(entries, afterIndex, missing) {
4604
+ const LOOKAHEAD = 6;
4605
+ const end = Math.min(entries.length, afterIndex + 1 + LOOKAHEAD);
4606
+ return missing.map(id => {
4607
+ for (let j = afterIndex + 1; j < end; j++) {
4608
+ const src = entries[j];
4609
+ const bi = src.content.findIndex(b => b.type === 'tool_result' && b.toolUseId === id);
4610
+ if (bi === -1)
4611
+ continue;
4612
+ const real = src.content[bi];
4613
+ const rest = src.content.filter((_, k) => k !== bi);
4614
+ src.content = rest.length > 0
4615
+ ? rest
4616
+ : [{ type: 'text', text: '[tool call omitted]' }];
4617
+ return real;
4618
+ }
4619
+ return {
4620
+ type: 'tool_result',
4621
+ toolUseId: id,
4622
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
4623
+ };
4624
+ });
4625
+ }
3348
4626
  /**
3349
4627
  * Prune tool_use / tool_result blocks in-place:
3350
4628
  * 1. Truncate `tool_use.input` blocks whose serialized JSON exceeds