@animalabs/context-manager 0.5.5 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/src/adaptive/kv-control.d.ts +69 -32
  2. package/dist/src/adaptive/kv-control.d.ts.map +1 -1
  3. package/dist/src/adaptive/kv-control.js +471 -172
  4. package/dist/src/adaptive/kv-control.js.map +1 -1
  5. package/dist/src/adaptive/picker.d.ts +8 -0
  6. package/dist/src/adaptive/picker.d.ts.map +1 -1
  7. package/dist/src/adaptive/picker.js.map +1 -1
  8. package/dist/src/adaptive/strategies/kv-stable.d.ts +12 -4
  9. package/dist/src/adaptive/strategies/kv-stable.d.ts.map +1 -1
  10. package/dist/src/adaptive/strategies/kv-stable.js +14 -5
  11. package/dist/src/adaptive/strategies/kv-stable.js.map +1 -1
  12. package/dist/src/context-log.d.ts.map +1 -1
  13. package/dist/src/context-log.js +11 -0
  14. package/dist/src/context-log.js.map +1 -1
  15. package/dist/src/context-manager.d.ts +28 -1
  16. package/dist/src/context-manager.d.ts.map +1 -1
  17. package/dist/src/context-manager.js +42 -0
  18. package/dist/src/context-manager.js.map +1 -1
  19. package/dist/src/index.d.ts +3 -2
  20. package/dist/src/index.d.ts.map +1 -1
  21. package/dist/src/index.js +1 -0
  22. package/dist/src/index.js.map +1 -1
  23. package/dist/src/message-store.d.ts +60 -0
  24. package/dist/src/message-store.d.ts.map +1 -1
  25. package/dist/src/message-store.js +165 -10
  26. package/dist/src/message-store.js.map +1 -1
  27. package/dist/src/strategies/autobiographical.d.ts +223 -2
  28. package/dist/src/strategies/autobiographical.d.ts.map +1 -1
  29. package/dist/src/strategies/autobiographical.js +1487 -142
  30. package/dist/src/strategies/autobiographical.js.map +1 -1
  31. package/dist/src/strategies/index.d.ts +1 -1
  32. package/dist/src/strategies/index.d.ts.map +1 -1
  33. package/dist/src/strategies/index.js.map +1 -1
  34. package/dist/src/strategies/knowledge.d.ts +8 -0
  35. package/dist/src/strategies/knowledge.d.ts.map +1 -1
  36. package/dist/src/strategies/knowledge.js +23 -1
  37. package/dist/src/strategies/knowledge.js.map +1 -1
  38. package/dist/src/types/strategy.d.ts +89 -5
  39. package/dist/src/types/strategy.d.ts.map +1 -1
  40. package/dist/src/types/strategy.js.map +1 -1
  41. package/dist/test/adaptive/doc-plus-chat.test.js +7 -3
  42. package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -1
  43. package/dist/test/adaptive/hard-fail.test.js +103 -0
  44. package/dist/test/adaptive/hard-fail.test.js.map +1 -1
  45. package/dist/test/adaptive/ingestion.test.js +3 -1
  46. package/dist/test/adaptive/ingestion.test.js.map +1 -1
  47. package/dist/test/adaptive/kv-control.test.js +17 -11
  48. package/dist/test/adaptive/kv-control.test.js.map +1 -1
  49. package/dist/test/adaptive/single-path-solve.test.d.ts +16 -0
  50. package/dist/test/adaptive/single-path-solve.test.d.ts.map +1 -0
  51. package/dist/test/adaptive/single-path-solve.test.js +204 -0
  52. package/dist/test/adaptive/single-path-solve.test.js.map +1 -0
  53. package/dist/test/anti-redundancy-budget.test.d.ts +17 -0
  54. package/dist/test/anti-redundancy-budget.test.d.ts.map +1 -0
  55. package/dist/test/anti-redundancy-budget.test.js +197 -0
  56. package/dist/test/anti-redundancy-budget.test.js.map +1 -0
  57. package/dist/test/chunk-persistence.test.d.ts +29 -0
  58. package/dist/test/chunk-persistence.test.d.ts.map +1 -0
  59. package/dist/test/chunk-persistence.test.js +290 -0
  60. package/dist/test/chunk-persistence.test.js.map +1 -0
  61. package/dist/test/compression-dedup.test.d.ts +20 -0
  62. package/dist/test/compression-dedup.test.d.ts.map +1 -0
  63. package/dist/test/compression-dedup.test.js +92 -0
  64. package/dist/test/compression-dedup.test.js.map +1 -0
  65. package/dist/test/compression-shape-invariants.test.js +40 -3
  66. package/dist/test/compression-shape-invariants.test.js.map +1 -1
  67. package/dist/test/head-window-order.test.d.ts +19 -0
  68. package/dist/test/head-window-order.test.d.ts.map +1 -0
  69. package/dist/test/head-window-order.test.js +159 -0
  70. package/dist/test/head-window-order.test.js.map +1 -0
  71. package/dist/test/image-strip-render-stats.test.js +25 -3
  72. package/dist/test/image-strip-render-stats.test.js.map +1 -1
  73. package/dist/test/knowledge-tool-boundary.test.d.ts +14 -0
  74. package/dist/test/knowledge-tool-boundary.test.d.ts.map +1 -0
  75. package/dist/test/knowledge-tool-boundary.test.js +93 -0
  76. package/dist/test/knowledge-tool-boundary.test.js.map +1 -0
  77. package/dist/test/l1-holdback.test.d.ts +15 -0
  78. package/dist/test/l1-holdback.test.d.ts.map +1 -0
  79. package/dist/test/l1-holdback.test.js +119 -0
  80. package/dist/test/l1-holdback.test.js.map +1 -0
  81. package/dist/test/merge-contiguity.test.d.ts +11 -0
  82. package/dist/test/merge-contiguity.test.d.ts.map +1 -0
  83. package/dist/test/merge-contiguity.test.js +82 -0
  84. package/dist/test/merge-contiguity.test.js.map +1 -0
  85. package/dist/test/message-store-window.test.d.ts +13 -0
  86. package/dist/test/message-store-window.test.d.ts.map +1 -0
  87. package/dist/test/message-store-window.test.js +177 -0
  88. package/dist/test/message-store-window.test.js.map +1 -0
  89. package/dist/test/speculation-cap.test.js +4 -0
  90. package/dist/test/speculation-cap.test.js.map +1 -1
  91. package/dist/test/strategy-persistence.test.js +41 -0
  92. package/dist/test/strategy-persistence.test.js.map +1 -1
  93. package/dist/test/summary-log-consistency.test.d.ts +17 -0
  94. package/dist/test/summary-log-consistency.test.d.ts.map +1 -0
  95. package/dist/test/summary-log-consistency.test.js +102 -0
  96. package/dist/test/summary-log-consistency.test.js.map +1 -0
  97. package/dist/test/tool-pairing-invariant.test.d.ts +15 -0
  98. package/dist/test/tool-pairing-invariant.test.d.ts.map +1 -0
  99. package/dist/test/tool-pairing-invariant.test.js +225 -0
  100. package/dist/test/tool-pairing-invariant.test.js.map +1 -0
  101. package/dist/tsconfig.tsbuildinfo +1 -1
  102. package/package.json +11 -3
  103. package/src/adaptive/kv-control.ts +509 -184
  104. package/src/adaptive/picker.ts +8 -0
  105. package/src/adaptive/strategies/kv-stable.ts +22 -10
  106. package/src/context-log.ts +11 -0
  107. package/src/context-manager.ts +48 -2
  108. package/src/index.ts +3 -2
  109. package/src/message-store.ts +202 -9
  110. package/src/strategies/autobiographical.ts +1576 -166
  111. package/src/strategies/index.ts +1 -1
  112. package/src/strategies/knowledge.ts +25 -1
  113. package/src/types/strategy.ts +97 -5
@@ -87,7 +87,12 @@ function formatInstruction(targetTokens: number): string {
87
87
  'Preserve concrete details — file paths, exact values, decisions, ' +
88
88
  `unresolved questions, the user\'s active asks. Target ~${targetTokens} ` +
89
89
  'tokens. Output only the memory body — no preamble, no section headers ' +
90
- 'unless they help preservation, no meta-commentary about summarizing.'
90
+ 'unless they help preservation, no meta-commentary about summarizing. ' +
91
+ 'Memorize only what actually happened in that slice: if it holds little ' +
92
+ 'beyond routine system traffic (heartbeats, empty turns, failure ' +
93
+ 'notices), a short memory saying so is correct — do not pad it by ' +
94
+ 're-narrating events you already remember from earlier as if they had ' +
95
+ 'just happened again.'
91
96
  );
92
97
  }
93
98
 
@@ -231,6 +236,36 @@ export interface Chunk {
231
236
  summaryId?: string;
232
237
  /** Phase type tag (set by KnowledgeStrategy for semantic chunking) */
233
238
  phaseType?: string;
239
+ /** ID of the persisted ChunkRecord backing this chunk (chunk persistence). */
240
+ recordId?: string;
241
+ }
242
+
243
+ /**
244
+ * Persisted chunk boundary, one per CLOSED chunk, stored in the
245
+ * `autobio:chunks` chronicle state slot (append_log, branch-aware).
246
+ *
247
+ * Records OWN the past: once a chunk closes, its membership is a persisted
248
+ * fact — rebuilds and restarts materialize chunks from records instead of
249
+ * recomputing boundaries from the running token sum. This is the fix for the
250
+ * 2026-07 re-consolidation storms: boundary inputs (config knobs, head
251
+ * window, token estimates, message mutations) could shift across restarts,
252
+ * the old exact-sourceIds-match recovery then failed, and whole stretches of
253
+ * already-summarized history were re-compressed into duplicate L1s.
254
+ *
255
+ * Membership is by message ID (never index), so edits/redactions degrade a
256
+ * record gracefully instead of re-keying its neighbors.
257
+ */
258
+ export interface ChunkRecord {
259
+ /** Stable record id ("c-<n>"). */
260
+ id: string;
261
+ /** Exact message IDs of the closed chunk, in order. */
262
+ sourceIds: string[];
263
+ /** Whether the chunk's L1 summary has been produced. */
264
+ compressed: boolean;
265
+ /** ID of the L1 SummaryEntry, once compressed. */
266
+ summaryId?: string;
267
+ /** Phase type tag (KnowledgeStrategy semantic chunking). */
268
+ phaseType?: string;
234
269
  }
235
270
 
236
271
  /**
@@ -269,6 +304,40 @@ function normalizePinLevels(opts?: PinLevelOptions): { level?: number; maxLevel?
269
304
  return maxLevel !== undefined ? { maxLevel } : {};
270
305
  }
271
306
 
307
+ /**
308
+ * Drop empty text blocks (`{type:'text', text:''}` or whitespace-only). The
309
+ * Anthropic API rejects them with 400 "text content blocks must be non-empty",
310
+ * which — thrown inside the speculative-compression drain — halts ALL
311
+ * compression. Non-text blocks (tool_use/tool_result/image) pass through
312
+ * unchanged, so tool pairing is preserved; callers drop any message left with
313
+ * an empty content array.
314
+ */
315
+ function stripEmptyTextBlocks(content: ContentBlock[]): ContentBlock[] {
316
+ return content.filter((b) => {
317
+ if (b.type !== 'text') return true;
318
+ const text = (b as { text?: unknown }).text;
319
+ return typeof text === 'string' && text.trim().length > 0;
320
+ });
321
+ }
322
+
323
+ /**
324
+ * Strip `thinking` / `redacted_thinking` blocks from compression INPUT.
325
+ *
326
+ * The summarizer must never be handed the agent's own reasoning:
327
+ * (a) signed thinking is only valid verbatim in the turn that produced it —
328
+ * replaying it into a rewritten summarize request corrupts the signature;
329
+ * (b) asking the model to summarize its own reasoning reads as reproducing /
330
+ * duplicating model output → `reasoning_extraction` refusal, which returns
331
+ * empty (→ "empty L1 summary, chunk left raw") or, worse, produces a
332
+ * summary that reproduces the reasoning as text — which then trips the
333
+ * SAME classifier on the MAIN thread once that summary is rendered.
334
+ * Thinking is scratch work, not history (the same rationale already applied to
335
+ * the summarizer's OUTPUT). Drop it from the input too.
336
+ */
337
+ function stripThinkingBlocks(content: ContentBlock[]): ContentBlock[] {
338
+ return content.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking');
339
+ }
340
+
272
341
  /**
273
342
  * Autobiographical chunking strategy.
274
343
  * Compresses old conversation chunks into summaries in the model's own words.
@@ -297,6 +366,32 @@ export class AutobiographicalStrategy implements ResettableStrategy {
297
366
  */
298
367
  protected _drainProgress = 0;
299
368
 
369
+ /**
370
+ * Chunks (keyed by their LAST message id — chunk membership is immutable
371
+ * once closed) whose L1 was explicitly demanded by a picker `produce` op.
372
+ * Demanded chunks bypass the `l1HoldbackChunks` window in `rebuildChunks`:
373
+ * speculation waits, demand doesn't. Entries become inert once the chunk
374
+ * compresses (compressed chunks are never re-queued), so no cleanup needed.
375
+ */
376
+ protected _demandedL1Chunks = new Set<MessageId>();
377
+
378
+ /**
379
+ * In-memory mirror of the persisted chunk records (autobio:chunks slot).
380
+ * Loaded in `loadPersistedState`, appended to when a chunk closes,
381
+ * updated by ID when its L1 lands.
382
+ */
383
+ protected chunkRecords: ChunkRecord[] = [];
384
+ protected chunkIdCounter = 0;
385
+ /**
386
+ * Fail-closed latch: set when most persisted records resolve to zero live
387
+ * messages (the messages-chain-break signature). While set, NO compression
388
+ * runs — duplicate memories are strictly worse than delayed compression.
389
+ */
390
+ protected chunkRecordsOrphaned = false;
391
+ private _orphanWarned = false;
392
+ /** Record ids whose compression was refused by the L1 overlap guard. */
393
+ private _overlapBlocked = new Set<string>();
394
+
300
395
  // Hierarchical state
301
396
  protected summaries: SummaryEntry[] = [];
302
397
  protected summaryIdCounter = 0;
@@ -313,11 +408,13 @@ export class AutobiographicalStrategy implements ResettableStrategy {
313
408
  /** Namespace for state-id scoping. Set in `initialize()`. */
314
409
  protected ns: string = '';
315
410
  protected get summariesStateId(): string { return `${this.ns}/autobio:summaries`; }
411
+ protected get chunksStateId(): string { return `${this.ns}/autobio:chunks`; }
316
412
  protected get counterStateId(): string { return `${this.ns}/autobio:counter`; }
317
413
  protected get mergeQueueStateId(): string { return `${this.ns}/autobio:mergeQueue`; }
318
414
  protected get pinsStateId(): string { return `${this.ns}/autobio:pins`; }
319
415
  protected get resolutionsStateId(): string { return `${this.ns}/autobio:resolutions`; }
320
416
  protected get locksStateId(): string { return `${this.ns}/autobio:locks`; }
417
+ protected get calibrationStateId(): string { return `${this.ns}/autobio:calibration`; }
321
418
 
322
419
  /** Protected ranges (pins + documents). Loaded from chronicle in initialize. */
323
420
  protected pins: ProtectedRange[] = [];
@@ -465,7 +562,117 @@ export class AutobiographicalStrategy implements ResettableStrategy {
465
562
  break;
466
563
  }
467
564
  }
565
+ // Legacy stores (pre chunk-persistence) have L1 summaries but no chunk
566
+ // records — synthesize records from L1 sourceIds before the first
567
+ // rebuild, so covered ground is owned and never re-compressed.
568
+ this.migrateChunkRecords(ctx.messageStore);
468
569
  this.rebuildChunks(ctx.messageStore);
570
+ // Kick the merge ladder for pre-existing unmerged summaries. Normally a
571
+ // compression/merge completion does this, but a store that boots with a
572
+ // backlog above threshold and an empty queue (e.g. after a pyramid
573
+ // repair pruned duplicates and un-merged survivors) would otherwise
574
+ // never start consolidating. Idempotent: already-queued/merged sources
575
+ // are skipped.
576
+ if (this.config.hierarchical && !this.chunkRecordsOrphaned) {
577
+ this.checkMergeThreshold();
578
+ }
579
+ }
580
+
581
+ /**
582
+ * Whether chunk boundaries are persisted to the `autobio:chunks` slot and
583
+ * own the past. Subclasses with their own chunking (KnowledgeStrategy's
584
+ * semantic phases) opt out and keep the legacy recompute-every-rebuild
585
+ * behavior.
586
+ */
587
+ protected get chunkPersistenceEnabled(): boolean {
588
+ return true;
589
+ }
590
+
591
+ /**
592
+ * One-time lazy migration: a store with L1 summaries but an empty chunks
593
+ * slot predates chunk persistence. Each L1's sourceIds ARE the historical
594
+ * chunk boundary — synthesize a compressed record per L1, in message
595
+ * order. Stale generations from the old partial-tail compression bug
596
+ * (an L1 whose messages are ALL already covered by records synthesized so
597
+ * far — prefix families, same-range duplicates) get NO record; coverage is
598
+ * what blocks re-compression, and the repair tooling prunes their content
599
+ * separately.
600
+ */
601
+ protected migrateChunkRecords(store: MessageStoreView): void {
602
+ if (!this.chunkPersistenceEnabled || !this.store) return;
603
+ if (this.chunkRecords.length > 0) return;
604
+ const l1s = this.summaries.filter(s => s.level === 1 && Array.isArray(s.sourceIds) && s.sourceIds.length > 0);
605
+ if (l1s.length === 0) return;
606
+
607
+ const msgIndex = new Map<string, number>();
608
+ store.getAll().forEach((m, i) => msgIndex.set(m.id, i));
609
+
610
+ // Sort by (start position asc, span length desc) so at each starting
611
+ // point the LONGEST generation claims the ground first.
612
+ const sorted = [...l1s].sort((a, b) => {
613
+ const sa = msgIndex.get(a.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
614
+ const sb = msgIndex.get(b.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
615
+ return sa - sb || b.sourceIds.length - a.sourceIds.length;
616
+ });
617
+
618
+ const covered = new Set<string>();
619
+ let skippedStale = 0;
620
+ let skippedGhost = 0;
621
+ for (const s of sorted) {
622
+ // An L1 none of whose messages exist anymore can't own live ground.
623
+ if (!s.sourceIds.some(id => msgIndex.has(id))) { skippedGhost++; continue; }
624
+ // Fully-covered = stale generation / contained duplicate.
625
+ if (s.sourceIds.every(id => covered.has(id))) { skippedStale++; continue; }
626
+ for (const id of s.sourceIds) covered.add(id);
627
+ this.appendChunkRecord({
628
+ id: `c-${this.chunkIdCounter++}`,
629
+ sourceIds: [...s.sourceIds],
630
+ compressed: true,
631
+ summaryId: s.id,
632
+ ...(s.phaseType ? { phaseType: s.phaseType } : {}),
633
+ });
634
+ }
635
+ console.warn(
636
+ `[autobiographical] chunk-persistence migration: ${l1s.length} L1s → ` +
637
+ `${this.chunkRecords.length} records (${skippedStale} stale generations, ` +
638
+ `${skippedGhost} fully-orphaned L1s skipped)`,
639
+ );
640
+ }
641
+
642
+ /** Append a record to the chunks slot + in-memory mirror. */
643
+ protected appendChunkRecord(record: ChunkRecord): void {
644
+ this.chunkRecords.push(record);
645
+ this.store?.appendToStateJson(this.chunksStateId, record);
646
+ }
647
+
648
+ /**
649
+ * Mark a chunk's record compressed, linking its L1. Resolves the log slot
650
+ * by ID against the persisted array — never by in-memory index (see
651
+ * setMergedInto for the clobber this avoids).
652
+ */
653
+ protected markChunkRecordCompressed(recordId: string | undefined, summaryId: string): void {
654
+ if (!this.chunkPersistenceEnabled || !recordId) return;
655
+ const rec = this.chunkRecords.find(r => r.id === recordId);
656
+ if (!rec) return;
657
+ rec.compressed = true;
658
+ rec.summaryId = summaryId;
659
+ if (!this.store) return;
660
+ const stored = this.store.getStateJson(this.chunksStateId);
661
+ if (!Array.isArray(stored)) return;
662
+ const payload = Buffer.from(JSON.stringify(rec));
663
+ let found = false;
664
+ for (let i = 0; i < stored.length; i++) {
665
+ const item = stored[i] as ChunkRecord | null;
666
+ if (item && item.id === recordId) {
667
+ this.store.editStateItem(this.chunksStateId, i, payload);
668
+ found = true;
669
+ }
670
+ }
671
+ if (!found) {
672
+ console.warn(
673
+ `[autobiographical] markChunkRecordCompressed: ${recordId} not found in persisted chunk log`,
674
+ );
675
+ }
469
676
  }
470
677
 
471
678
  /**
@@ -483,6 +690,16 @@ export class AutobiographicalStrategy implements ResettableStrategy {
483
690
  fullSnapshotEvery: 10,
484
691
  });
485
692
  } catch { /* already registered */ }
693
+ if (this.chunkPersistenceEnabled) {
694
+ try {
695
+ this.store.registerState({
696
+ id: this.chunksStateId,
697
+ strategy: 'append_log',
698
+ deltaSnapshotEvery: 50,
699
+ fullSnapshotEvery: 10,
700
+ });
701
+ } catch { /* already registered */ }
702
+ }
486
703
  try {
487
704
  this.store.registerState({
488
705
  id: this.counterStateId,
@@ -501,6 +718,12 @@ export class AutobiographicalStrategy implements ResettableStrategy {
501
718
  strategy: 'snapshot',
502
719
  });
503
720
  } catch { /* already registered */ }
721
+ try {
722
+ this.store.registerState({
723
+ id: this.calibrationStateId,
724
+ strategy: 'snapshot',
725
+ });
726
+ } catch { /* already registered */ }
504
727
  // Adaptive-resolution state slots — only registered when the flag is on
505
728
  // so chronicles without the flag don't accumulate unused slots.
506
729
  if (this.config.adaptiveResolution) {
@@ -531,10 +754,91 @@ export class AutobiographicalStrategy implements ResettableStrategy {
531
754
  this.mergeQueue = [];
532
755
  this.pins = [];
533
756
  this.pinIdCounter = 0;
757
+ this.chunkRecords = [];
758
+ this.chunkIdCounter = 0;
534
759
  return;
535
760
  }
761
+
762
+ if (this.chunkPersistenceEnabled) {
763
+ const records = this.store.getStateJson(this.chunksStateId);
764
+ this.chunkRecords = (Array.isArray(records) ? (records as ChunkRecord[]) : [])
765
+ .filter(r => r && typeof r.id === 'string' && Array.isArray(r.sourceIds) && r.sourceIds.length > 0);
766
+ this.chunkIdCounter = this.chunkRecords.reduce((max, r) => {
767
+ const n = Number(r.id.replace(/^c-/, ''));
768
+ return Number.isFinite(n) && n >= max ? n + 1 : max;
769
+ }, 0);
770
+ this.chunkRecordsOrphaned = false;
771
+ this._orphanWarned = false;
772
+ this._overlapBlocked.clear();
773
+ }
536
774
  const summaries = this.store.getStateJson(this.summariesStateId);
537
- this.summaries = Array.isArray(summaries) ? (summaries as SummaryEntry[]) : [];
775
+ const loaded = Array.isArray(summaries) ? (summaries as SummaryEntry[]) : [];
776
+ // Drop empty-content summaries (bugged/empty generations from before the
777
+ // production guards). Recalling or merging one yields an empty text block →
778
+ // Anthropic 400 "content must be non-empty". Never let them re-enter memory.
779
+ const nonEmpty = loaded.filter(s => s && typeof s.content === 'string' && s.content.trim().length > 0);
780
+ const droppedEmpty = loaded.length - nonEmpty.length;
781
+ const removedEmptyIds = new Set(
782
+ loaded
783
+ .filter(s => s && (typeof s.content !== 'string' || s.content.trim().length === 0))
784
+ .map(s => s.id),
785
+ );
786
+ if (droppedEmpty > 0) console.warn(`[autobiographical] dropped ${droppedEmpty} empty summary(ies) on load`);
787
+ // Dedupe by id, keeping the copy with mergedInto set (position of first
788
+ // occurrence preserved). Duplicate-id copies with diverging merge state
789
+ // exist in stores touched by the pre-fix setMergedInto index-desync bug;
790
+ // without dedupe, the plain copy stays on the unmerged frontier and its
791
+ // content renders twice (once itself, once via its parent's merge).
792
+ const byId = new Map<string, SummaryEntry>();
793
+ for (const s of nonEmpty) {
794
+ const prev = byId.get(s.id);
795
+ if (!prev) byId.set(s.id, s);
796
+ else if (!prev.mergedInto && s.mergedInto) byId.set(s.id, s);
797
+ }
798
+ const dupes = nonEmpty.length - byId.size;
799
+ if (dupes > 0) console.warn(`[autobiographical] deduped ${dupes} duplicate summary id(s) on load`);
800
+ // Dropping an invalid parent is only half the repair. Its children may
801
+ // still carry `mergedInto: <dropped-id>`, which makes them simultaneously
802
+ // unavailable to the picker (no parent to render) and ineligible for a
803
+ // replacement merge (they still look merged). Clear every dangling edge
804
+ // and persist the canonicalized array so the poison does not return on
805
+ // every restart.
806
+ let danglingParents = 0;
807
+ this.summaries = [...byId.values()].map((summary) => {
808
+ if (!summary.mergedInto || byId.has(summary.mergedInto)) return summary;
809
+ danglingParents++;
810
+ const { mergedInto: _dropped, ...repaired } = summary;
811
+ return repaired as SummaryEntry;
812
+ });
813
+ if (droppedEmpty > 0 || dupes > 0 || danglingParents > 0) {
814
+ this.store.setStateJson(this.summariesStateId, this.summaries);
815
+ console.warn(
816
+ `[autobiographical] repaired summary state: removed ${droppedEmpty} empty, ` +
817
+ `deduped ${dupes}, cleared ${danglingParents} dangling parent pointer(s)`,
818
+ );
819
+ }
820
+
821
+ // An invalid L1 may also be referenced by a persisted chunk record. Make
822
+ // that chunk compressible again instead of leaving it permanently marked
823
+ // complete with no summary behind it.
824
+ if (this.chunkPersistenceEnabled && this.chunkRecords.length > 0) {
825
+ const validL1Ids = new Set(this.summaries.filter(s => s.level === 1).map(s => s.id));
826
+ let repairedChunkRecords = 0;
827
+ this.chunkRecords = this.chunkRecords.map((record) => {
828
+ if (!record.compressed || (record.summaryId && validL1Ids.has(record.summaryId))) {
829
+ return record;
830
+ }
831
+ repairedChunkRecords++;
832
+ const { summaryId: _dropped, ...rest } = record;
833
+ return { ...rest, compressed: false };
834
+ });
835
+ if (repairedChunkRecords > 0) {
836
+ this.store.setStateJson(this.chunksStateId, this.chunkRecords);
837
+ console.warn(
838
+ `[autobiographical] repaired ${repairedChunkRecords} chunk record(s) with missing L1 summaries`,
839
+ );
840
+ }
841
+ }
538
842
 
539
843
  const counter = this.store.getStateJson(this.counterStateId);
540
844
  this.summaryIdCounter = typeof counter === 'number' ? counter : 0;
@@ -543,6 +847,15 @@ export class AutobiographicalStrategy implements ResettableStrategy {
543
847
  this.mergeQueue = Array.isArray(queue)
544
848
  ? (queue as Array<{ level: SummaryLevel; sourceIds: string[] }>)
545
849
  : [];
850
+ const validMergeQueue = this.mergeQueue.filter(
851
+ merge => !merge.sourceIds.some(id => removedEmptyIds.has(id) && !byId.has(id)),
852
+ );
853
+ if (validMergeQueue.length !== this.mergeQueue.length) {
854
+ const removed = this.mergeQueue.length - validMergeQueue.length;
855
+ this.mergeQueue = validMergeQueue;
856
+ this.store.setStateJson(this.mergeQueueStateId, this.mergeQueue);
857
+ console.warn(`[autobiographical] removed ${removed} merge queue item(s) with missing sources`);
858
+ }
546
859
 
547
860
  const pinsState = this.store.getStateJson(this.pinsStateId);
548
861
  if (pinsState && typeof pinsState === 'object' && Array.isArray((pinsState as { pins?: unknown }).pins)) {
@@ -813,6 +1126,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
813
1126
  * Single point so subclasses inherit persistence.
814
1127
  */
815
1128
  protected pushSummary(entry: SummaryEntry): void {
1129
+ if (typeof entry.content !== 'string' || entry.content.trim().length === 0) {
1130
+ throw new Error(
1131
+ `[autobiographical] refusing to persist empty summary ${entry.id} at L${entry.level}`,
1132
+ );
1133
+ }
816
1134
  this.summaries.push(entry);
817
1135
  this.store?.appendToStateJson(this.summariesStateId, entry);
818
1136
  }
@@ -857,13 +1175,30 @@ export class AutobiographicalStrategy implements ResettableStrategy {
857
1175
  protected setMergedInto(entry: SummaryEntry, mergedIntoId: string): void {
858
1176
  entry.mergedInto = mergedIntoId;
859
1177
  if (!this.store) return;
860
- const index = this.summaries.indexOf(entry);
861
- if (index < 0) return;
862
- this.store.editStateItem(
863
- this.summariesStateId,
864
- index,
865
- Buffer.from(JSON.stringify(entry)),
866
- );
1178
+ // Resolve the log position by ID against the PERSISTED array — never by
1179
+ // in-memory index. `loadPersistedState` filters empty-content summaries
1180
+ // out of `this.summaries` while they remain in the log, so after a reload
1181
+ // the in-memory index is shifted relative to the log slot. Editing by
1182
+ // in-memory index wrote merge-updates onto NEIGHBORING entries, silently
1183
+ // clobbering them (4 summaries lost in the 2026-07 Lena incident, leaving
1184
+ // duplicate-id copies with diverging mergedInto). Update every stored
1185
+ // copy with this id so past duplicates converge too.
1186
+ const stored = this.store.getStateJson(this.summariesStateId);
1187
+ if (!Array.isArray(stored)) return;
1188
+ let found = false;
1189
+ const payload = Buffer.from(JSON.stringify(entry));
1190
+ for (let i = 0; i < stored.length; i++) {
1191
+ const item = stored[i] as SummaryEntry | null;
1192
+ if (item && item.id === entry.id) {
1193
+ this.store.editStateItem(this.summariesStateId, i, payload);
1194
+ found = true;
1195
+ }
1196
+ }
1197
+ if (!found) {
1198
+ console.warn(
1199
+ `[autobiographical] setMergedInto: ${entry.id} not found in persisted summary log — merge state not persisted`,
1200
+ );
1201
+ }
867
1202
  }
868
1203
 
869
1204
  /**
@@ -959,6 +1294,10 @@ export class AutobiographicalStrategy implements ResettableStrategy {
959
1294
  }
960
1295
  for (const chunk of candidates) {
961
1296
  if (chunk.compressed) continue;
1297
+ // Demand path: mark the chunk so the l1HoldbackChunks window in
1298
+ // rebuildChunks never filters it back out of the queue.
1299
+ const lastId = chunk.messages[chunk.messages.length - 1]?.id;
1300
+ if (lastId !== undefined) this._demandedL1Chunks.add(lastId);
962
1301
  if (this.compressionQueue.includes(chunk.index)) continue;
963
1302
  this.compressionQueue.push(chunk.index);
964
1303
  }
@@ -1280,6 +1619,14 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1280
1619
  // snapshot, so it reflects what the last compile actually rendered rather than
1281
1620
  // re-deriving the full pyramid (which, under adaptive resolution, bears little
1282
1621
  // resemblance to the folded output).
1622
+ //
1623
+ // CAVEAT: the final structural passes (`trimOrphanedToolUse`,
1624
+ // `enforceToolPairing`) run AFTER the per-entry tallies and BEFORE `rsEnd()`,
1625
+ // and are NOT reflected in the snapshot. Trims only remove entries (total
1626
+ // over-counts by at most the trimmed tail); the pairing validator can also
1627
+ // ADD stub tool_result entries (total then under-counts by those stubs).
1628
+ // Both deltas are a handful of tokens — the stats describe the selection, not
1629
+ // the byte-exact wire payload. Do not treat `total` as an exact wire count.
1283
1630
  // ===========================================================================
1284
1631
  private _rs: RenderStats | null = null;
1285
1632
  private _lastRenderStats: RenderStats | null = null;
@@ -1413,16 +1760,20 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1413
1760
  msgCap: number,
1414
1761
  maxTokens: number,
1415
1762
  totalTokensBefore: number,
1763
+ // Post-strip per-message estimates (2026-07-12): eviction must price a
1764
+ // message the way the stripped render will. Raw pricing counted every
1765
+ // stripped image at full cost and evicted most of an image-era tail the
1766
+ // recent-window walk had correctly admitted (mythos: 601-message tail
1767
+ // admitted, 228 survived eviction, 45k rendered of a 120k window).
1768
+ pse?: number[],
1416
1769
  ): { messages: number; tokens: number } {
1417
1770
  if (recentStart >= messages.length) return { messages: 0, tokens: 0 };
1418
1771
 
1772
+ const est = (i: number): number => pse?.[i] ?? store.estimateTokens(messages[i]);
1419
1773
  const accepted: number[] = [];
1420
1774
  let acceptedTokens = 0;
1421
1775
  for (let i = messages.length - 1; i >= recentStart; i--) {
1422
- const msg = messages[i];
1423
- const tokens = msgCap > 0
1424
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1425
- : store.estimateTokens(msg);
1776
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1426
1777
  if (this.isOverBudget(totalTokensBefore + acceptedTokens + tokens, maxTokens)) break;
1427
1778
  accepted.push(i);
1428
1779
  acceptedTokens += tokens;
@@ -1442,9 +1793,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1442
1793
  for (const i of accepted) {
1443
1794
  const msg = messages[i];
1444
1795
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
1445
- const tokens = msgCap > 0
1446
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1447
- : store.estimateTokens(msg);
1796
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1448
1797
  entries.push({
1449
1798
  index: entries.length,
1450
1799
  sourceMessageId: msg.id,
@@ -1508,30 +1857,194 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1508
1857
  throw new Error('No membrane instance for compression');
1509
1858
  }
1510
1859
 
1860
+ // ---- Duplicate-formation guards (layered) ----
1861
+ // Merged from two independent fixes for the same disease:
1862
+ //
1863
+ // 1. EXACT MATCH → adopt (bug 6.10, Tengro). rebuildChunks (fired by
1864
+ // onNewMessage / select) can re-queue a span whose compression already
1865
+ // completed against a stale chunk object — or, under chunk
1866
+ // persistence, a crash between the L1 append and the record edit
1867
+ // leaves a record uncompressed with its summary already in the log.
1868
+ // Adopt the existing summary, skip the LLM call, and heal the record.
1869
+ const chunkIdKey = this.chunkKey(chunk);
1870
+ const exactExisting = this.summaries.find(
1871
+ s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey
1872
+ );
1873
+ if (exactExisting) {
1874
+ chunk.compressed = true;
1875
+ chunk.summaryId = exactExisting.id;
1876
+ this.markChunkRecordCompressed(chunk.recordId, exactExisting.id);
1877
+ return;
1878
+ }
1879
+
1880
+ const coveredByL1 = new Set<string>();
1881
+ for (const s of this.summaries) {
1882
+ if (s.level === 1 && Array.isArray(s.sourceIds)) {
1883
+ for (const id of s.sourceIds) coveredByL1.add(id);
1884
+ }
1885
+ }
1886
+
1887
+ // 2. FULLY COVERED (non-exact) → drop rather than duplicate history
1888
+ // (bug 6.10, Tengro): boundaries shifted across a rebuild and every
1889
+ // message is already inside some L1. Marking compressed WITHOUT a
1890
+ // summaryId means the uncompressed-middle fallback skips these
1891
+ // messages — they render only via the covering L1s. Safe while
1892
+ // `recentStart` advances monotonically (a fully-covered OLD chunk
1893
+ // can't intersect the recent-exclusion window); if that assumption is
1894
+ // ever weakened, reinstate a raw fallback (chunk-level `coveredBy`)
1895
+ // rather than dropping. The chunk record (if any) is deliberately
1896
+ // left uncompressed as an operator breadcrumb.
1897
+ if (chunk.messages.length > 0 && chunk.messages.every(m => coveredByL1.has(m.id))) {
1898
+ console.warn(
1899
+ `[autobiographical] dedup guard: chunk ${chunk.recordId ?? `#${chunk.index}`} ` +
1900
+ `is fully covered by existing L1s under different boundaries — dropped, not re-compressed.`,
1901
+ );
1902
+ chunk.compressed = true;
1903
+ return;
1904
+ }
1905
+
1906
+ // 3. PARTIAL OVERLAP → refuse (strict; chunk persistence). With
1907
+ // close-then-compress there is NO legitimate way for a chunk to
1908
+ // partially overlap an existing L1's span. If it happens anyway
1909
+ // (bookkeeping bug, store surgery gone wrong), refuse to produce:
1910
+ // a warning in the log is strictly better than a duplicate memory
1911
+ // in an agent's head.
1912
+ const overlapIds = chunk.messages.filter(m => coveredByL1.has(m.id)).map(m => m.id);
1913
+ if (overlapIds.length > 0) {
1914
+ const key = chunk.recordId ?? chunkIdKey;
1915
+ if (!this._overlapBlocked.has(key)) {
1916
+ this._overlapBlocked.add(key);
1917
+ console.error(
1918
+ `[autobiographical] OVERLAP GUARD: refusing to compress chunk ` +
1919
+ `${chunk.recordId ?? `#${chunk.index}`} — ${overlapIds.length}/${chunk.messages.length} ` +
1920
+ `of its messages are already covered by existing L1 summaries ` +
1921
+ `(first: ${overlapIds[0]}). Duplicate-memory formation blocked; ` +
1922
+ `investigate before resuming this span.`,
1923
+ );
1924
+ }
1925
+ return;
1926
+ }
1927
+
1511
1928
  const targetTokens = this.config.summaryTargetTokens ?? 2000;
1512
1929
  const agentParticipant = this.config.summaryParticipant ?? 'Claude';
1513
1930
 
1931
+ // ---- 0. Thin-chunk guard ----
1932
+ // A chunk of silent/skip turns and bare system traffic gives the
1933
+ // summarizer nothing to remember. Asked anyway, it confabulates: it
1934
+ // reaches for the nearest salient content (head window, prior recall
1935
+ // pairs) and narrates it as if it just happened — each such L1 then
1936
+ // compounds through merges (the "68 initiations" incident). Store a
1937
+ // mechanical stub without an LLM call instead. Chunks with any
1938
+ // non-text blocks (tool cycles, images) are never stubbed.
1939
+ const minChunkChars = this.config.minChunkCharsForLLM ?? 200;
1940
+ if (minChunkChars > 0) {
1941
+ let substantiveChars = 0;
1942
+ let hasNonText = false;
1943
+ for (const m of chunk.messages) {
1944
+ for (const b of m.content) {
1945
+ if (b.type === 'text') substantiveChars += b.text.trim().length;
1946
+ else hasNonText = true;
1947
+ }
1948
+ }
1949
+ if (!hasNonText && substantiveChars < minChunkChars) {
1950
+ const messageIds = chunk.messages.map(m => m.id);
1951
+ const stub: SummaryEntry = {
1952
+ id: `L1-${this.nextSummaryIdCounter()}`,
1953
+ level: 1,
1954
+ content:
1955
+ `(A quiet stretch: ${chunk.messages.length} messages of routine ` +
1956
+ `system traffic — heartbeats, empty turns, notices. Nothing ` +
1957
+ `happened worth remembering.)`,
1958
+ tokens: 40,
1959
+ sourceLevel: 0,
1960
+ sourceIds: messageIds,
1961
+ sourceRange: { first: messageIds[0], last: messageIds[messageIds.length - 1] },
1962
+ created: Date.now(),
1963
+ phaseType: chunk.phaseType,
1964
+ };
1965
+ this.pushSummary(stub);
1966
+ chunk.compressed = true;
1967
+ chunk.summaryId = stub.id;
1968
+ this.markChunkRecordCompressed(chunk.recordId, stub.id);
1969
+ this._compressionCount++;
1970
+ logCompressionCall({
1971
+ operation: 'compress_l1',
1972
+ system: null,
1973
+ messages: [],
1974
+ metadata: {
1975
+ stub: true,
1976
+ chunk_message_ids: messageIds,
1977
+ chunk_size: chunk.messages.length,
1978
+ substantive_chars: substantiveChars,
1979
+ min_chunk_chars: minChunkChars,
1980
+ summary_id: stub.id,
1981
+ },
1982
+ response: stub.content,
1983
+ });
1984
+ this.checkMergeThreshold();
1985
+ return;
1986
+ }
1987
+ }
1988
+
1514
1989
  // Build the KV-preserving prompt per hermes-autobio spec:
1515
1990
  //
1516
- // 1. Prior summaries narrativized as CM-asks / agent-recalls
1991
+ // 1. Headthe raw chronicle opening (identity anchor), FIRST,
1992
+ // exactly where the original instance saw it. It MUST precede
1993
+ // the recall pairs: when it followed them (pre-2026-07 order),
1994
+ // it read as the most recent live conversation, and for thin
1995
+ // chunks the summarizer narrated the head as fresh events
1996
+ // ("Antra came to me to explore the transformation story
1997
+ // again…"), compounding across merges into runaway false
1998
+ // memories (the "68 initiations" incident). Chronological
1999
+ // order is also the KV-stable order — the head never changes.
2000
+ // (executeMerge always had head-first; this site was the odd
2001
+ // one out.)
2002
+ // 2. Prior summaries — narrativized as CM-asks / agent-recalls
1517
2003
  // pairs, in source order. The unmerged frontier of the
1518
2004
  // summary forest: any summary that has not yet been merged
1519
2005
  // into a higher level. After merges run, the L_{k+1} replaces
1520
2006
  // its L_k children — using the children plus their parent
1521
2007
  // doubles the prompt size unboundedly.
1522
- // 2. Headraw messages before the chunk that aren't already
1523
- // represented by a prior summary.
1524
- // 3. Marker — in-band signal that a memory is about to form.
1525
- // 4. Chunk — raw messages being compressed, as the agent
2008
+ // 3. Raw middle — messages between head and chunk not covered by
2009
+ // any summary (usually empty).
2010
+ // 4. Marker — in-band signal that a memory is about to form.
2011
+ // 5. Chunk — raw messages being compressed, as the agent
1526
2012
  // experienced them.
1527
- // 5. Instruction — doc-aware if the chunk is part of a bodyGroup.
2013
+ // 6. Instruction — doc-aware if the chunk is part of a bodyGroup.
1528
2014
  //
1529
2015
  // There is intentionally NO tail_after_chunk: that would leak
1530
2016
  // future information into the model's KV state and corrupt the
1531
2017
  // as-of framing of memory formation.
1532
2018
  const llmMessages: Array<{ participant: string; content: ContentBlock[] }> = [];
1533
2019
 
1534
- // ---- 1. Prior recall pairs ----
2020
+ // ---- 1. Head window (raw, ALWAYS present, FIRST) ----
2021
+ //
2022
+ // The head is the foundational identity anchor: the actual opening
2023
+ // of the chronicle (the user's first message, the agent's first
2024
+ // reply, the system context if any). It establishes WHO is speaking
2025
+ // to WHOM. Without it, when the chunk content is heavily first-person
2026
+ // from someone other than the agent (e.g., a user-shared document),
2027
+ // the agent loses its first-person grounding and drifts into the
2028
+ // content author's voice.
2029
+ //
2030
+ // The head is the configured head window — not "everything before
2031
+ // the chunk." For doc-heavy chronicles, "everything before" would
2032
+ // be hundreds of thousands of tokens; the recall pairs below
2033
+ // represent that intermediate content. The head is just the
2034
+ // permanent prefix that the original instance always saw.
2035
+ //
2036
+ // Head messages are excluded from compression by `getCompressibleMessages`
2037
+ // (they're outside the chunking range), so they won't appear in
2038
+ // any L1's sourceIds — no overlap with the recall pairs below.
2039
+ const allMessages = ctx.messageStore.getAll();
2040
+ const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
2041
+ const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
2042
+ for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
2043
+ const m = allMessages[i];
2044
+ llmMessages.push({ participant: m.participant, content: m.content });
2045
+ }
2046
+
2047
+ // ---- 2. Prior recall pairs ----
1535
2048
  // Filter to the unmerged frontier: any summary whose `mergedInto`
1536
2049
  // is unset. After merge, the children's mergedInto points at the
1537
2050
  // parent and the parent stands alone with that source range. The
@@ -1540,7 +2053,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1540
2053
  // converged to ~500 L1s that never aged out, blowing the 200k
1541
2054
  // window around chunk 118.
1542
2055
  const priorSummaries = this.summaries
1543
- .filter((s) => !s.mergedInto)
2056
+ // Skip empty-content summaries: emitting `{type:'text', text:''}` as a
2057
+ // recall pair triggers Anthropic 400 "text content blocks must be
2058
+ // non-empty", which stalls ALL compression (mirrors the render-path guard
2059
+ // + load-drop). A single empty summary otherwise poisons every compression.
2060
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1544
2061
  .sort((a, b) => a.sourceRange.first.localeCompare(b.sourceRange.first));
1545
2062
 
1546
2063
  // Token-budget cap (see capRecallPairs). Defense-in-depth: even with
@@ -1577,33 +2094,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1577
2094
  });
1578
2095
  }
1579
2096
 
1580
- // ---- 2. Head window (raw, ALWAYS present) ----
1581
- //
1582
- // The head is the foundational identity anchor: the actual opening
1583
- // of the chronicle (the user's first message, the agent's first
1584
- // reply, the system context if any). It establishes WHO is speaking
1585
- // to WHOM. Without it, when the chunk content is heavily first-person
1586
- // from someone other than the agent (e.g., a user-shared document),
1587
- // the agent loses its first-person grounding and drifts into the
1588
- // content author's voice.
1589
- //
1590
- // The head is the configured head window — not "everything before
1591
- // the chunk." For doc-heavy chronicles, "everything before" would
1592
- // be hundreds of thousands of tokens; the recall pairs already
1593
- // represent that intermediate content. The head is just the
1594
- // permanent prefix that the original instance always saw.
1595
- //
1596
- // Head messages are excluded from compression by `getCompressibleMessages`
1597
- // (they're outside the chunking range), so they won't appear in
1598
- // any L1's sourceIds — no overlap with the recall pairs above.
1599
- const allMessages = ctx.messageStore.getAll();
1600
- const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1601
- const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1602
- for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1603
- const m = allMessages[i];
1604
- llmMessages.push({ participant: m.participant, content: m.content });
1605
- }
1606
-
2097
+ // ---- 3. Raw middle ----
1607
2098
  // Any raw messages between the head and the chunk that aren't yet
1608
2099
  // represented by any summary — usually empty in adaptive-resolution
1609
2100
  // mode, since chunking proceeds contiguously and summaries cover
@@ -1635,18 +2126,18 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1635
2126
  }
1636
2127
  }
1637
2128
 
1638
- // ---- 3. In-band marker ----
2129
+ // ---- 4. In-band marker ----
1639
2130
  llmMessages.push({
1640
2131
  participant: 'Context Manager',
1641
2132
  content: [{ type: 'text', text: COMPRESSION_MARKER }],
1642
2133
  });
1643
2134
 
1644
- // ---- 4. Chunk messages raw ----
2135
+ // ---- 5. Chunk messages raw ----
1645
2136
  for (const m of chunk.messages) {
1646
2137
  llmMessages.push({ participant: m.participant, content: m.content });
1647
2138
  }
1648
2139
 
1649
- // ---- 5. Instruction (reading-mode aware) ----
2140
+ // ---- 6. Instruction (reading-mode aware) ----
1650
2141
  //
1651
2142
  // When the chunk is a portion of a substantially larger sharded message
1652
2143
  // (≥ 2× chunk size), use the reading-mode instruction. It avoids the
@@ -1680,6 +2171,18 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1680
2171
  // by its tool_result, and any tool_result that doesn't follow a use.
1681
2172
  const cleaned = stripUnpairedToolBlocks(collapsed);
1682
2173
 
2174
+ // Without the agent's live tool definitions, a request that replays
2175
+ // tool-block-bearing history is deterministically refused
2176
+ // (reasoning_extraction -- see the `tools` comment below). Tools are
2177
+ // pushed by the host on every activation; before the first activation
2178
+ // of a session, defer rather than burn a doomed full-window call.
2179
+ const chunkHasToolBlocks = cleaned.some(m =>
2180
+ m.content.some((b: ContentBlock) => b.type === 'tool_use' || b.type === 'tool_result'));
2181
+ if (chunkHasToolBlocks && !(ctx.tools && ctx.tools.length > 0)) {
2182
+ console.warn('[autobiographical] deferring chunk compression: history contains tool blocks but host has not provided tool definitions yet (ctx.tools empty) — will retry after next activation');
2183
+ return;
2184
+ }
2185
+
1683
2186
  // NO system prompt. The agent's identity is established by the head
1684
2187
  // (the actual conversation opening — user message + agent reply that
1685
2188
  // grounded the original instance). A system prompt would (a) add a
@@ -1688,12 +2191,53 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1688
2191
  // (b) provide an alternative identity source that competes with the
1689
2192
  // structural one carried by the conversation itself. Anchoring
1690
2193
  // identity by the chronicle's actual head is more honest.
2194
+ // Own the byte wall here rather than delegating to membrane's shed: cap
2195
+ // the prompt's inline image bytes newest-first before the request is built.
2196
+ // A tighter budget than the live window's: a compression prompt also
2197
+ // carries the head, the whole recall frontier and the raw chunk, so the
2198
+ // image share must leave room for all of it under the API's 32MB cap.
2199
+ this.capCompressionImageBytes(
2200
+ llmMessages as Array<{ content: ContentBlock[] }>,
2201
+ this.config.maxCompressionImageBytes ??
2202
+ AutobiographicalStrategy.DEFAULT_MAX_COMPRESSION_IMAGE_BYTES,
2203
+ );
2204
+
1691
2205
  const request: NormalizedRequest = {
1692
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
2206
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
2207
+ // raw history that can carry more inline image bytes than the API's
2208
+ // request cap. Dropping the OLDEST images from the summarizer's view is
2209
+ // acceptable policy here — the summary describes the span, it does not
2210
+ // preserve pixels — and membrane error-logs every exercised shed. All
2211
+ // other callers fail loudly instead (no silent transport mutation).
2212
+ shedOversizeImages: true,
2213
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
2214
+ // message left with no content. An empty-content turn (e.g. a silent/skip
2215
+ // turn that produced no text) otherwise reaches the API as an empty text
2216
+ // block → 400 "text content blocks must be non-empty", which throws in the
2217
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
2218
+ // recall-header guard — together they cover every source of the 400.)
2219
+ messages: cleaned
2220
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
2221
+ .filter(m => m.content.length > 0),
1693
2222
  config: {
1694
2223
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1695
- maxTokens: Math.round(targetTokens * 1.5),
2224
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
2225
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
2226
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
2227
+ // off rich memories (stop=max_tokens).
2228
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
1696
2229
  },
2230
+ // Declare the agent's live tools. A summarizer request that replays
2231
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
2232
+ // safety classifier as a foreign agent trace being duplicated ->
2233
+ // deterministic reasoning_extraction refusal of every memory-write
2234
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
2235
+ // tools the live instance runs with is also strictly MORE faithful to
2236
+ // the original context, so it is the KV-honest choice, not a
2237
+ // workaround. Undefined before the first activation of a session --
2238
+ // acceptable: those chunks stay raw and are retried after the agent's
2239
+ // first turn (see the defer guard in compressChunkHierarchical).
2240
+ tools: ctx.tools,
1697
2241
  };
1698
2242
 
1699
2243
  const callStart = Date.now();
@@ -1713,12 +2257,45 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1713
2257
  .join('\n');
1714
2258
  logResponse = summaryText;
1715
2259
 
2260
+ // A bugged/empty generation (summarizer returned no text — spent budget on
2261
+ // thinking, truncated, etc.) must NOT be stored: recalled later it becomes
2262
+ // an empty assistant text block → Anthropic 400 "content must be non-empty".
2263
+ // Leave the chunk raw rather than poisoning memory with an empty summary.
2264
+ if (!summaryText.trim()) {
2265
+ console.warn(`[autobiographical] empty L1 summary for chunk of ${chunk.messages.length} msgs — skipping (chunk left raw)`);
2266
+ return;
2267
+ }
2268
+
2269
+ // Re-check the dedup guard AFTER the await: summary state may have
2270
+ // changed while the LLM call was in flight (persisted-state reload,
2271
+ // or any future concurrent producer). Discarding a paid-for result
2272
+ // is cheaper than storing a duplicate L1 over the same messages.
2273
+ const postExisting = this.summaries.find(
2274
+ s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey
2275
+ );
2276
+ if (postExisting) {
2277
+ chunk.compressed = true;
2278
+ chunk.summaryId = postExisting.id;
2279
+ return;
2280
+ }
2281
+
1716
2282
  const messageIds = chunk.messages.map(m => m.id);
1717
2283
  const entry: SummaryEntry = {
1718
2284
  id: `L1-${this.nextSummaryIdCounter()}`,
1719
2285
  level: 1,
1720
2286
  content: summaryText,
1721
- tokens: Math.ceil(summaryText.length / 4),
2287
+ // Exact when available (2026-07-12): the compression response's own
2288
+ // usage.outputTokens IS the true token count of the text it just
2289
+ // wrote — the single most-reused number in the pyramid (fold floor,
2290
+ // middle budget, recall caps). Estimate only as fallback.
2291
+ tokens:
2292
+ response.usage?.outputTokens &&
2293
+ response.usage.outputTokens > 0 &&
2294
+ // outputTokens includes scratch thinking when present — only exact
2295
+ // when the whole response is the summary text itself.
2296
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
2297
+ ? response.usage.outputTokens
2298
+ : Math.ceil(summaryText.length / 3),
1722
2299
  sourceLevel: 0,
1723
2300
  sourceIds: messageIds,
1724
2301
  sourceRange: {
@@ -1732,6 +2309,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1732
2309
  this.pushSummary(entry);
1733
2310
  chunk.compressed = true;
1734
2311
  chunk.summaryId = entry.id;
2312
+ this.markChunkRecordCompressed(chunk.recordId, entry.id);
1735
2313
  this._compressionCount++;
1736
2314
  logSummaryId = entry.id;
1737
2315
 
@@ -1781,6 +2359,78 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1781
2359
  * already-eligible siblings, producing N near-identical higher-level
1782
2360
  * summaries when the queue eventually drains.
1783
2361
  */
2362
+ /**
2363
+ * Pick merge sources as the oldest CONTIGUOUS run (2026-07-12 fix).
2364
+ *
2365
+ * The old rule merged "whatever N are unmerged" in creation order. On a
2366
+ * store whose frontier held both June L2s and a July L2, that minted an
2367
+ * L3 spanning months of already-merged history (mythos L3-415, 0-3853 of
2368
+ * 3995 messages) — a group whose range straddles the recent window can
2369
+ * never fold (group-atomicity vs the raw zone), so the whole deep lineage
2370
+ * above it went unusable and the fold floor stopped fitting the budget.
2371
+ *
2372
+ * Rules: order candidates by live source position; break runs where the
2373
+ * positional gap exceeds `mergeContiguityGapLimit` (holes from wiped/
2374
+ * pruned nodes are fine, cross-era bridges are not); exclude candidates
2375
+ * whose OWN span exceeds `mergeMaxSourceSpanMessages` (replay-era wide-
2376
+ * span summaries would bridge anything they join); merge the oldest run
2377
+ * that still has `threshold` members.
2378
+ */
2379
+ protected contiguousMergeCandidates(
2380
+ unmerged: SummaryEntry[],
2381
+ threshold: number,
2382
+ ): SummaryEntry[] | null {
2383
+ if (unmerged.length < threshold) return null;
2384
+ const messageOrder = new Map<MessageId, number>();
2385
+ let seq = 0;
2386
+ for (const ch of this.chunks) {
2387
+ for (const m of ch.messages) messageOrder.set(m.id, seq++);
2388
+ }
2389
+ const gapLimit = this.config.mergeContiguityGapLimit ?? 300;
2390
+ const spanLimit = this.config.mergeMaxSourceSpanMessages ?? 1500;
2391
+ const withPos: Array<{ s: SummaryEntry; first: number; last: number }> = [];
2392
+ for (const s of unmerged) {
2393
+ const first = messageOrder.get(s.sourceRange.first);
2394
+ const last = messageOrder.get(s.sourceRange.last);
2395
+ if (first === undefined || last === undefined) continue;
2396
+ if (Math.abs(last - first) > spanLimit) continue; // wide-span quarantine
2397
+ withPos.push({ s, first: Math.min(first, last), last: Math.max(first, last) });
2398
+ }
2399
+ withPos.sort((a, b) => a.first - b.first);
2400
+
2401
+ // Split into contiguous runs (a gap larger than `gapLimit` starts a new one).
2402
+ const runs: Array<typeof withPos> = [];
2403
+ let run: typeof withPos = [];
2404
+ let runEnd = -Infinity;
2405
+ for (const x of withPos) {
2406
+ if (run.length > 0 && x.first - runEnd > gapLimit) {
2407
+ runs.push(run);
2408
+ run = [];
2409
+ runEnd = -Infinity;
2410
+ }
2411
+ run.push(x);
2412
+ runEnd = Math.max(runEnd, x.last);
2413
+ }
2414
+ if (run.length > 0) runs.push(run);
2415
+ if (runs.length === 0) return null;
2416
+
2417
+ // ONLY THE NEWEST RUN CAN GROW (2026-07-12 starvation fix). Summaries are
2418
+ // always produced at the live end, so any INTERIOR run is stranded: it can
2419
+ // never reach `threshold` members, and waiting for it froze the whole
2420
+ // pyramid (mythos: L1 frontier [913-981]x5 + [4039-4131]x5 separated by a
2421
+ // 3058-message hole from the poison-node surgery — 10 unmerged L1s, 8
2422
+ // unmerged L2s, merge queue empty, fold floor climbing to 117k until the
2423
+ // picker died). Interior runs consolidate as soon as they have 2 members;
2424
+ // only the newest run waits for the full threshold.
2425
+ for (let i = 0; i < runs.length; i++) {
2426
+ const r = runs[i];
2427
+ const isNewest = i === runs.length - 1;
2428
+ if (r.length >= threshold) return r.slice(0, threshold).map((x) => x.s);
2429
+ if (!isNewest && r.length >= 2) return r.slice(0, threshold).map((x) => x.s);
2430
+ }
2431
+ return null;
2432
+ }
2433
+
1784
2434
  protected checkMergeThreshold(): void {
1785
2435
  phaseChannel.report('merge-threshold'); // liveness-watchdog phase
1786
2436
  if (this.config.speculativeProduction) {
@@ -1803,11 +2453,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1803
2453
  const unmergedL1 = this.summaries.filter(
1804
2454
  s => s.level === 1 && !s.mergedInto && !queuedL1.has(s.id),
1805
2455
  );
1806
- if (unmergedL1.length >= threshold) {
1807
- const toMerge = unmergedL1.slice(0, threshold);
2456
+ const l1Run = this.contiguousMergeCandidates(unmergedL1, threshold);
2457
+ if (l1Run) {
1808
2458
  this.enqueueMerge({
1809
2459
  level: 2,
1810
- sourceIds: toMerge.map(s => s.id),
2460
+ sourceIds: l1Run.map(s => s.id),
1811
2461
  });
1812
2462
  }
1813
2463
 
@@ -1815,11 +2465,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1815
2465
  const unmergedL2 = this.summaries.filter(
1816
2466
  s => s.level === 2 && !s.mergedInto && !queuedL2.has(s.id),
1817
2467
  );
1818
- if (unmergedL2.length >= threshold) {
1819
- const toMerge = unmergedL2.slice(0, threshold);
2468
+ const l2Run = this.contiguousMergeCandidates(unmergedL2, threshold);
2469
+ if (l2Run) {
1820
2470
  this.enqueueMerge({
1821
2471
  level: 3,
1822
- sourceIds: toMerge.map(s => s.id),
2472
+ sourceIds: l2Run.map(s => s.id),
1823
2473
  });
1824
2474
  }
1825
2475
  }
@@ -1863,11 +2513,11 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1863
2513
  const unmerged = this.summaries.filter(
1864
2514
  s => s.level === level && !getSummaryParentId(s) && !queued.has(s.id),
1865
2515
  );
1866
- if (unmerged.length >= threshold) {
1867
- const toMerge = unmerged.slice(0, threshold);
2516
+ const run = this.contiguousMergeCandidates(unmerged, threshold);
2517
+ if (run) {
1868
2518
  this.enqueueMerge({
1869
2519
  level: level + 1,
1870
- sourceIds: toMerge.map(s => s.id),
2520
+ sourceIds: run.map(s => s.id),
1871
2521
  });
1872
2522
  }
1873
2523
  }
@@ -1987,7 +2637,9 @@ export class AutobiographicalStrategy implements ResettableStrategy {
1987
2637
  // parent — the same rule used everywhere else and now in
1988
2638
  // `compressChunkHierarchical`. The cap below is the defense-in-depth.
1989
2639
  const priorSummariesAll = this.summaries
1990
- .filter((s) => !s.mergedInto)
2640
+ // Skip empty-content summaries (see compressChunkHierarchical): an empty
2641
+ // text block in the merge recall header 400s and stalls compression.
2642
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1991
2643
  .filter((s) => {
1992
2644
  for (const lid of s.sourceIds) if (sourceLeafIds.has(lid)) return false;
1993
2645
  const firstIdx = allMessages.findIndex((m) => m.id === s.sourceRange.first);
@@ -2147,15 +2799,56 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2147
2799
  const collapsed = this.collapseConsecutiveMessages(split);
2148
2800
  const cleaned = stripUnpairedToolBlocks(collapsed);
2149
2801
 
2802
+ // Byte wall on the MERGE prompt too (2026-07-12). A merge expands its
2803
+ // sources ONE LEVEL DEEPER — an L2 merge therefore replays the RAW
2804
+ // messages under its L1s, images and all (including screenshots nested in
2805
+ // tool_results). This is the path that kept tripping membrane's transport
2806
+ // shed at 27MB after the L1 site was already capped. Own it here.
2807
+ this.capCompressionImageBytes(
2808
+ cleaned as Array<{ content: ContentBlock[] }>,
2809
+ this.config.maxCompressionImageBytes ??
2810
+ AutobiographicalStrategy.DEFAULT_MAX_COMPRESSION_IMAGE_BYTES,
2811
+ );
2812
+
2150
2813
  // NO system prompt — identity is established by the head window
2151
2814
  // (present at the start of llmMessages above) and by the prior
2152
2815
  // recall pairs. Same rationale as compressChunkHierarchical.
2153
2816
  const request: NormalizedRequest = {
2154
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
2817
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
2818
+ // raw history that can carry more inline image bytes than the API's
2819
+ // request cap. Dropping the OLDEST images from the summarizer's view is
2820
+ // acceptable policy here — the summary describes the span, it does not
2821
+ // preserve pixels — and membrane error-logs every exercised shed. All
2822
+ // other callers fail loudly instead (no silent transport mutation).
2823
+ shedOversizeImages: true,
2824
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
2825
+ // message left with no content. An empty-content turn (e.g. a silent/skip
2826
+ // turn that produced no text) otherwise reaches the API as an empty text
2827
+ // block → 400 "text content blocks must be non-empty", which throws in the
2828
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
2829
+ // recall-header guard — together they cover every source of the 400.)
2830
+ messages: cleaned
2831
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
2832
+ .filter(m => m.content.length > 0),
2155
2833
  config: {
2156
2834
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
2157
- maxTokens: Math.round(targetTokens * 1.5),
2835
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
2836
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
2837
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
2838
+ // off rich memories (stop=max_tokens).
2839
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
2158
2840
  },
2841
+ // Declare the agent's live tools. A summarizer request that replays
2842
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
2843
+ // safety classifier as a foreign agent trace being duplicated ->
2844
+ // deterministic reasoning_extraction refusal of every memory-write
2845
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
2846
+ // tools the live instance runs with is also strictly MORE faithful to
2847
+ // the original context, so it is the KV-honest choice, not a
2848
+ // workaround. Undefined before the first activation of a session --
2849
+ // acceptable: those chunks stay raw and are retried after the agent's
2850
+ // first turn (see the defer guard in compressChunkHierarchical).
2851
+ tools: ctx.tools,
2159
2852
  };
2160
2853
 
2161
2854
  const callStart = Date.now();
@@ -2172,6 +2865,14 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2172
2865
  .join('\n');
2173
2866
  logResponse = mergedText;
2174
2867
 
2868
+ // Empty merged generation: skip the merge entirely (do NOT push or mark
2869
+ // sources merged) so we never store/recall an empty summary. Sources stay
2870
+ // unmerged and can be retried.
2871
+ if (!mergedText.trim()) {
2872
+ console.warn(`[autobiographical] empty merged L${targetLevel} summary (${sources.length} sources) — skipping merge`);
2873
+ return;
2874
+ }
2875
+
2175
2876
  // Compute source range from constituent summaries
2176
2877
  const sourceRange = {
2177
2878
  first: sources[0].sourceRange.first,
@@ -2183,7 +2884,15 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2183
2884
  id: `L${targetLevel}-${this.nextSummaryIdCounter()}`,
2184
2885
  level: targetLevel,
2185
2886
  content: mergedText,
2186
- tokens: Math.ceil(mergedText.length / 4),
2887
+ // Exact when available — see the L1 site.
2888
+ tokens:
2889
+ response.usage?.outputTokens &&
2890
+ response.usage.outputTokens > 0 &&
2891
+ // outputTokens includes scratch thinking when present — only exact
2892
+ // when the whole response is the summary text itself.
2893
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
2894
+ ? response.usage.outputTokens
2895
+ : Math.ceil(mergedText.length / 3),
2187
2896
  sourceLevel,
2188
2897
  sourceIds,
2189
2898
  sourceRange,
@@ -2254,10 +2963,18 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2254
2963
  this.rsBegin();
2255
2964
  const entries: ContextEntry[] = [];
2256
2965
  const maxTokens = budget.maxTokens - budget.reserveForResponse;
2966
+ const overBudgetGraceRatio = Math.max(0, this.config.overBudgetGraceRatio ?? 0);
2967
+ const rejectionBudget = Math.floor(maxTokens * (1 + overBudgetGraceRatio));
2968
+ // Closed-loop calibration: apply the persisted multiplier BEFORE any
2969
+ // estimate is taken this compile.
2970
+ this.loadCalibration(store);
2257
2971
  const messages = store.getAll();
2258
2972
  const msgCap = this.config.maxMessageTokens;
2973
+ // Post-strip estimates (see postStripEstimates): every budgeting site in
2974
+ // this method prices a message the way the stripped render will cost it.
2975
+ const pse = this.postStripEstimates(store);
2259
2976
 
2260
- // ----- 1. Build head/tail sets and emit head entries -----
2977
+ // ----- 1. Build head/tail sets and reserve the tail before emitting -----
2261
2978
  const headStart = this.getHeadWindowStartIndex(store);
2262
2979
  const headEnd = this.getHeadWindowEnd(store);
2263
2980
  const recentStart = this.getRecentWindowStart(store);
@@ -2267,14 +2984,45 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2267
2984
  let headTokens = 0;
2268
2985
  let tailTokens = 0;
2269
2986
 
2987
+ // Compute the fixed raw windows first. They are hard reservations, not
2988
+ // best-effort phases: foldable history may use only the space left after
2989
+ // every head and tail message has been accounted for.
2990
+ for (let i = headStart; i < headEnd && i < messages.length; i++) {
2991
+ const msg = messages[i];
2992
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2993
+ headMessageIds.add(msg.id);
2994
+ headTokens += tokens;
2995
+ }
2996
+ const effectiveRecentStart = Math.max(recentStart, headEnd);
2997
+ for (let i = effectiveRecentStart; i < messages.length; i++) {
2998
+ const msg = messages[i];
2999
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
3000
+ tailMessageIds.add(msg.id);
3001
+ tailTokens += tokens;
3002
+ }
3003
+
3004
+ if (headTokens + tailTokens > rejectionBudget) {
3005
+ throw new OverBudgetError({
3006
+ budget: rejectionBudget,
3007
+ actual: headTokens + tailTokens,
3008
+ diagnostics: {
3009
+ headTokens,
3010
+ tailTokens,
3011
+ middleTokens: 0,
3012
+ middleChunkCount: Math.max(0, effectiveRecentStart - headEnd),
3013
+ deepestLevel: 0,
3014
+ },
3015
+ });
3016
+ }
3017
+
3018
+ const prefixBudget = rejectionBudget - tailTokens;
2270
3019
  let totalTokens = 0;
2271
3020
 
2272
- // Emit head entries verbatim
3021
+ // Emit the already-reserved head entries verbatim.
2273
3022
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
2274
3023
  const msg = messages[i];
2275
3024
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2276
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2277
- if (totalTokens + tokens > maxTokens) break;
3025
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2278
3026
  entries.push({
2279
3027
  index: entries.length,
2280
3028
  sourceMessageId: msg.id,
@@ -2283,23 +3031,12 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2283
3031
  content,
2284
3032
  });
2285
3033
  totalTokens += tokens;
2286
- headMessageIds.add(msg.id);
2287
- headTokens += tokens;
2288
3034
  this.rsRaw('head', tokens);
2289
3035
  }
2290
3036
  // (Cache breakpoints are placed in one pass over the FINAL ordered entries
2291
3037
  // below — see placeCacheMarkers — capturing the stable folded prefix, not
2292
3038
  // just the head boundary.)
2293
3039
 
2294
- // Compute tail message IDs (will be emitted at end)
2295
- const effectiveRecentStart = Math.max(recentStart, headEnd);
2296
- for (let i = effectiveRecentStart; i < messages.length; i++) {
2297
- const msg = messages[i];
2298
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2299
- tailMessageIds.add(msg.id);
2300
- tailTokens += tokens;
2301
- }
2302
-
2303
3040
  // ----- 2. Build PickerChunks for messages in the middle -----
2304
3041
  // For each compressible (non-head, non-tail) message we create one
2305
3042
  // PickerChunk. Its l1Id is determined by the existing chunks that
@@ -2330,8 +3067,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2330
3067
  const msg = messages[i];
2331
3068
  const ch = chunksByMessageId.get(msg.id);
2332
3069
  const tokens = msgCap > 0
2333
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2334
- : store.estimateTokens(msg);
3070
+ ? Math.min(pse[i], msgCap + 50)
3071
+ : pse[i];
2335
3072
  const bound = pinBounds.get(i);
2336
3073
  pickerChunks.push({
2337
3074
  id: msg.id,
@@ -2345,6 +3082,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2345
3082
  pinLevel: bound?.level,
2346
3083
  pinMaxLevel: bound?.maxLevel,
2347
3084
  l1Id: ch?.summaryId,
3085
+ salience: AutobiographicalStrategy.staticSalience(msg),
2348
3086
  });
2349
3087
  }
2350
3088
 
@@ -2353,8 +3091,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2353
3091
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
2354
3092
  const msg = messages[i];
2355
3093
  const tokens = msgCap > 0
2356
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2357
- : store.estimateTokens(msg);
3094
+ ? Math.min(pse[i], msgCap + 50)
3095
+ : pse[i];
2358
3096
  pickerChunks.push({
2359
3097
  id: msg.id,
2360
3098
  sequence: i,
@@ -2368,8 +3106,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2368
3106
  for (let i = effectiveRecentStart; i < messages.length; i++) {
2369
3107
  const msg = messages[i];
2370
3108
  const tokens = msgCap > 0
2371
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2372
- : store.estimateTokens(msg);
3109
+ ? Math.min(pse[i], msgCap + 50)
3110
+ : pse[i];
2373
3111
  pickerChunks.push({
2374
3112
  id: msg.id,
2375
3113
  sequence: i,
@@ -2421,6 +3159,17 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2421
3159
  const picker = this.buildPicker(pickerInputs);
2422
3160
  const result = picker.run(pickerInputs, foldingBudget);
2423
3161
 
3162
+ // Every trust-region override is loud (design §13.4) — silence was half
3163
+ // of the 2026-07-12 incident.
3164
+ const plan = this._lastKvStable?.lastPlan();
3165
+ if (plan?.override) {
3166
+ console.error(
3167
+ `[kv-escalation] override=${plan.override} perturbation=${plan.perturbation}` +
3168
+ ` tokens=${plan.tokens} budget=${foldingBudget.totalBudget}` +
3169
+ ` (see adaptive-resolution-design.md §13.4)`,
3170
+ );
3171
+ }
3172
+
2424
3173
  // Commit the new resolutions back to strategy state for next compile.
2425
3174
  // Persist to chronicle only if anything actually changed — avoids
2426
3175
  // unnecessary state-slot writes on no-op compiles (which is the common
@@ -2456,14 +3205,12 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2456
3205
  this.handleProducedOps(result.produced);
2457
3206
  }
2458
3207
 
2459
- // Hard-fail check: if the picker exhausted itself but the final render
2460
- // would still exceed the HARD budget (not just the soft target), surface
2461
- // an OverBudgetError to the host rather than silently dropping entries.
2462
- // The strategy has done all it can; the application has to decide what
2463
- // to do next (raise budget, switch model, drop windows, etc.).
2464
- if (result.exhausted && result.finalTokens > totalBudget) {
3208
+ // Hard-fail whenever the picker's current plan exceeds the hard budget.
3209
+ // A `produce` op only schedules a missing summary; it does not make the
3210
+ // current raw plan feasible and must never authorize an inference.
3211
+ if (result.finalTokens > rejectionBudget) {
2465
3212
  throw new OverBudgetError({
2466
- budget: totalBudget,
3213
+ budget: rejectionBudget,
2467
3214
  actual: result.finalTokens,
2468
3215
  diagnostics: {
2469
3216
  headTokens,
@@ -2535,7 +3282,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2535
3282
  // (`maxMessageTokens` is for per-message caps on chat / tool
2536
3283
  // results, not for sharded bodyGroup composites.)
2537
3284
  const tokens = this.estimateTokens(content);
2538
- if (totalTokens + tokens > maxTokens) {
3285
+ if (totalTokens + tokens > prefixBudget) {
2539
3286
  currentRun = null;
2540
3287
  return false;
2541
3288
  }
@@ -2567,7 +3314,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2567
3314
  sourceRelation: 'derived',
2568
3315
  };
2569
3316
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2570
- if (totalTokens + pairTokens > maxTokens) {
3317
+ if (totalTokens + pairTokens > prefixBudget) {
2571
3318
  currentRun = null;
2572
3319
  return false;
2573
3320
  }
@@ -2625,8 +3372,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2625
3372
  const resolution = result.finalResolutions.get(msg.id) ?? 0;
2626
3373
  if (resolution === 0) {
2627
3374
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2628
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2629
- if (totalTokens + tokens > maxTokens) break;
3375
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
3376
+ if (totalTokens + tokens > prefixBudget) break;
2630
3377
  entries.push({
2631
3378
  index: entries.length,
2632
3379
  sourceMessageId: msg.id,
@@ -2641,8 +3388,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2641
3388
  const ancestor = this.findAncestorAt(msg.id, resolution, chunksByMessageId, summariesById);
2642
3389
  if (!ancestor) {
2643
3390
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2644
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2645
- if (totalTokens + tokens > maxTokens) break;
3391
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
3392
+ if (totalTokens + tokens > prefixBudget) break;
2646
3393
  entries.push({
2647
3394
  index: entries.length,
2648
3395
  sourceMessageId: msg.id,
@@ -2674,7 +3421,7 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2674
3421
  sourceRelation: 'derived',
2675
3422
  };
2676
3423
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2677
- if (totalTokens + pairTokens > maxTokens) break;
3424
+ if (totalTokens + pairTokens > prefixBudget) break;
2678
3425
  entries.push(questionEntry);
2679
3426
  entries.push(answerEntry);
2680
3427
  totalTokens += pairTokens;
@@ -2683,8 +3430,21 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2683
3430
  }
2684
3431
  }
2685
3432
 
2686
- // ----- 6. Emit tail entries newest-first eviction (matches existing behavior) -----
2687
- const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
3433
+ // ----- 6. Emit the fully-reserved tail -----
3434
+ const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, rejectionBudget, totalTokens, pse);
3435
+ if (tailStats.messages !== messages.length - effectiveRecentStart) {
3436
+ throw new OverBudgetError({
3437
+ budget: rejectionBudget,
3438
+ actual: totalTokens + tailTokens,
3439
+ diagnostics: {
3440
+ headTokens,
3441
+ tailTokens,
3442
+ middleTokens: totalTokens - headTokens,
3443
+ middleChunkCount: pickerChunks.length - headMessageIds.size - tailMessageIds.size,
3444
+ deepestLevel,
3445
+ },
3446
+ });
3447
+ }
2688
3448
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
2689
3449
 
2690
3450
  // ----- 7. Post-process: merge consecutive raw entries from the same bodyGroup -----
@@ -2697,9 +3457,43 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2697
3457
 
2698
3458
  this.pruneToolEntries(merged);
2699
3459
  this.trimOrphanedToolUse(merged);
3460
+ // Full pairing invariant over the final rendered context — catches the
3461
+ // mid-list orphans the trailing/leading trims can't (bug 6.7). The adaptive
3462
+ // path has the same mid-list orphan producers as hierarchical (budget
3463
+ // `break`s between pair members, raw emission interleaved with recall
3464
+ // pairs), and FKM defaults autobiographical strategies onto this path — so
3465
+ // the guard has to run here too. It's a no-op on already-valid output.
3466
+ this.enforceToolPairing(merged);
2700
3467
  // Strip stale images BEFORE placing markers and committing stats, so both
2701
3468
  // describe the post-strip context the agent actually receives.
2702
3469
  this.applyImageStripping(merged, store);
3470
+
3471
+ // The newest stored message is the triggering turn for this compile. A
3472
+ // structural repair may rewrite tool blocks, but it must never erase that
3473
+ // turn. Body-group shards merge under the first shard's source id, so any
3474
+ // surviving member of the same group proves the newest shard is present.
3475
+ const newest = messages[messages.length - 1];
3476
+ if (newest) {
3477
+ const newestGroupIds = newest.bodyGroupId
3478
+ ? new Set(messages.filter(m => m.bodyGroupId === newest.bodyGroupId).map(m => m.id))
3479
+ : new Set([newest.id]);
3480
+ const newestRetained = merged.some(
3481
+ entry => entry.sourceMessageId && newestGroupIds.has(entry.sourceMessageId),
3482
+ );
3483
+ if (!newestRetained) {
3484
+ throw new OverBudgetError({
3485
+ budget: rejectionBudget,
3486
+ actual: totalTokens + tailTokens,
3487
+ diagnostics: {
3488
+ headTokens,
3489
+ tailTokens,
3490
+ middleTokens: totalTokens - headTokens,
3491
+ middleChunkCount: pickerChunks.length - headMessageIds.size - tailMessageIds.size,
3492
+ deepestLevel,
3493
+ },
3494
+ });
3495
+ }
3496
+ }
2703
3497
  // Place ≤4 cache breakpoints across the FINAL ordered entries so the
2704
3498
  // provider can reuse the stable folded prefix — not just the head. With a
2705
3499
  // single head marker the cache hit is ~2%; well-placed breakpoints take the
@@ -2707,9 +3501,100 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2707
3501
  // placement is the dominant KV lever).
2708
3502
  this.placeCacheMarkers(merged, headMessageIds, tailMessageIds);
2709
3503
  this.rsEnd();
3504
+ // Closed-loop calibration bookkeeping: the committed render stats total
3505
+ // (in CURRENT calibrated units) is what this compile claims the request
3506
+ // will cost — reportRealInputTokens compares provider usage against it.
3507
+ this._storeView = store;
3508
+ const rs = this.getRenderStats(store);
3509
+ this._lastCompileEstimate =
3510
+ rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens +
3511
+ rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens;
3512
+ this._calibrationArmed = true; // exactly one sample per compile
2710
3513
  return merged;
2711
3514
  }
2712
3515
 
3516
+ private _lastCompileEstimate = 0;
3517
+ private _storeView: MessageStoreView | null = null;
3518
+
3519
+ /**
3520
+ * Closed-loop estimator calibration (2026-07-12). Feed the REAL input total
3521
+ * for a request built from the latest compile (fresh + cache_read +
3522
+ * cache_creation, minus non-window overhead the caller knows about, e.g.
3523
+ * tool schemas). Maintains an EMA of real/estimated and applies it as the
3524
+ * store's global multiplier, persisted across restarts. The per-class rates
3525
+ * carry the shape; this carries the residual level.
3526
+ */
3527
+ reportRealInputTokens(realTotal: number): void {
3528
+ if (!Number.isFinite(realTotal) || realTotal <= 0) return;
3529
+ const est = this._lastCompileEstimate;
3530
+ if (!est || est <= 0) return;
3531
+
3532
+ // ARM-ONCE-PER-COMPILE (2026-07-12 regression fix). A turn makes MANY API
3533
+ // calls — tool-use rounds and max_tokens continuations each append to the
3534
+ // request — so only the FIRST completion after a compile was built from
3535
+ // the window this estimate describes. Feeding later calls compares a grown
3536
+ // request against the original estimate: ratios of 2.0-2.3 (est=224k
3537
+ // real=504k) drove the multiplier 1.0 -> 2.37 in minutes, inflating every
3538
+ // estimate (the fold floor went 62k -> 108k on unchanged content) and
3539
+ // exhausting the picker on every wake. One sample per compile, always.
3540
+ if (!this._calibrationArmed) return;
3541
+ this._calibrationArmed = false;
3542
+
3543
+ const ratio = realTotal / est;
3544
+ // SANITY BAND: a representative sample sits near 1. Anything wilder is a
3545
+ // structural mismatch (a request we didn't compile, a partial compile, a
3546
+ // provider quirk) — never evidence about chars-per-token. Log, don't learn.
3547
+ if (ratio < 0.6 || ratio > 1.8) {
3548
+ console.error(
3549
+ `[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ` +
3550
+ `(est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k) — ` +
3551
+ `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}`,
3552
+ );
3553
+ return;
3554
+ }
3555
+
3556
+ const current = this._calibration;
3557
+ const observed = ratio * current; // back out the multiplier already applied
3558
+ const alpha = 0.2; // slow EMA: one wild request shouldn't yank the ruler
3559
+ const next = current + alpha * (observed - current);
3560
+ const clamped = Math.min(1.8, Math.max(0.6, next));
3561
+ if (Math.abs(clamped - current) / current > 0.02) {
3562
+ console.error(
3563
+ `[estimator-calibration] real/est=${ratio.toFixed(2)} ` +
3564
+ `multiplier ${current.toFixed(2)} -> ${clamped.toFixed(2)} (est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k)`,
3565
+ );
3566
+ }
3567
+ this._calibration = clamped;
3568
+ this.applyCalibration();
3569
+ try {
3570
+ this.store?.setStateJson(this.calibrationStateId, { multiplier: this._calibration, at: Date.now() });
3571
+ } catch { /* persistence is best-effort */ }
3572
+ }
3573
+
3574
+ private _calibrationArmed = false;
3575
+
3576
+ private _calibration = 1;
3577
+ private _calibrationLoaded = false;
3578
+
3579
+ protected applyCalibration(): void {
3580
+ this._storeView?.setTokenCalibration?.(this._calibration);
3581
+ }
3582
+
3583
+ /** Load the persisted multiplier once and push it into the store view. */
3584
+ protected loadCalibration(store: MessageStoreView): void {
3585
+ this._storeView = store;
3586
+ if (!this._calibrationLoaded) {
3587
+ this._calibrationLoaded = true;
3588
+ try {
3589
+ const saved = this.store?.getStateJson(this.calibrationStateId) as { multiplier?: number } | null;
3590
+ if (saved && Number.isFinite(saved.multiplier)) {
3591
+ this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier!));
3592
+ }
3593
+ } catch { /* absent slot is fine */ }
3594
+ }
3595
+ this.applyCalibration();
3596
+ }
3597
+
2713
3598
  /**
2714
3599
  * Place up to four `cache_control` breakpoints across the final ordered
2715
3600
  * entries: the head/system boundary, the end of the folded history (the
@@ -2763,12 +3648,23 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2763
3648
  ): ContextEntry[] {
2764
3649
  if (entries.length === 0) return entries;
2765
3650
 
2766
- // Look up bodyGroupId by sourceMessageId via the message store.
2767
- const groupOf = (sourceMessageId?: string): string | undefined => {
3651
+ // Look up bodyGroup metadata by sourceMessageId, memoized for the
3652
+ // duration of this pass. Entries reference the same messages repeatedly
3653
+ // (run-extension checks + the shardIndex sort comparator below), and
3654
+ // store.get() also resolves blobs — fetch each message at most once.
3655
+ const shardMeta = new Map<string, { groupId?: string; shardIndex?: number }>();
3656
+ const metaOf = (sourceMessageId?: string): { groupId?: string; shardIndex?: number } | undefined => {
2768
3657
  if (!sourceMessageId) return undefined;
2769
- const m = store.get(sourceMessageId);
2770
- return m?.bodyGroupId;
3658
+ let meta = shardMeta.get(sourceMessageId);
3659
+ if (!meta) {
3660
+ const m = store.get(sourceMessageId);
3661
+ meta = { groupId: m?.bodyGroupId, shardIndex: m?.shardIndex };
3662
+ shardMeta.set(sourceMessageId, meta);
3663
+ }
3664
+ return meta;
2771
3665
  };
3666
+ const groupOf = (sourceMessageId?: string): string | undefined =>
3667
+ metaOf(sourceMessageId)?.groupId;
2772
3668
 
2773
3669
  const out: ContextEntry[] = [];
2774
3670
  let i = 0;
@@ -2800,9 +3696,9 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2800
3696
  // ordering. (Head/tail emission keeps chronological order, but defending
2801
3697
  // against reorderings is cheap.)
2802
3698
  const sortedRun = [...run].sort((a, b) => {
2803
- const ma = a.sourceMessageId ? store.get(a.sourceMessageId) : null;
2804
- const mb = b.sourceMessageId ? store.get(b.sourceMessageId) : null;
2805
- return (ma?.shardIndex ?? 0) - (mb?.shardIndex ?? 0);
3699
+ const sa = metaOf(a.sourceMessageId)?.shardIndex ?? 0;
3700
+ const sb = metaOf(b.sourceMessageId)?.shardIndex ?? 0;
3701
+ return sa - sb;
2806
3702
  });
2807
3703
  // Build merged text content. Non-text blocks (rare in shards) are
2808
3704
  // preserved on the first shard's entry only.
@@ -2848,16 +3744,89 @@ export class AutobiographicalStrategy implements ResettableStrategy {
2848
3744
  */
2849
3745
  protected buildPicker(inputs: PickerInputs): Picker {
2850
3746
  if (this.config.foldingStrategy === 'kv-stable') {
2851
- return new Picker(
2852
- new KvStableStrategy(inputs, {
2853
- reachTokens: this.config.kvStableReachTokens,
2854
- mergeThreshold: this.config.mergeThreshold,
2855
- }),
2856
- );
3747
+ const strategy = new KvStableStrategy(inputs, {
3748
+ reachTokens: this.config.kvStableReachTokens,
3749
+ qualityGapRatio: this.config.kvStableQualityGapRatio,
3750
+ mergeThreshold: this.config.mergeThreshold,
3751
+ });
3752
+ this._lastKvStable = strategy;
3753
+ return new Picker(strategy);
2857
3754
  }
3755
+ this._lastKvStable = null;
2858
3756
  return this.getAdaptivePicker();
2859
3757
  }
2860
3758
 
3759
+ /** The kv-stable strategy instance behind the most recent compile — kept for
3760
+ * `[kv-escalation]` observability (design §13.4: every override is loud). */
3761
+ private _lastKvStable: KvStableStrategy | null = null;
3762
+
3763
+ /**
3764
+ * Static salience prior (design §13.3) — "is the window the only copy?".
3765
+ * Content whose payload is externalized folds cheap: tool blocks
3766
+ * (re-derivable), fenced code (usually written to disk), images (the file/
3767
+ * CDN keeps them), bare link drops. Conversation exists nowhere but the
3768
+ * chronicle, so it stays at 1. Returns a value in [0.2, 1]; cheap,
3769
+ * deterministic, computed per message at picker-input construction.
3770
+ */
3771
+ protected static staticSalience(msg: StoredMessage): number {
3772
+ let totalChars = 0;
3773
+ let externalChars = 0;
3774
+ for (const block of msg.content) {
3775
+ const b = block as {
3776
+ type?: string;
3777
+ text?: string;
3778
+ input?: unknown;
3779
+ content?: unknown;
3780
+ };
3781
+ switch (b.type) {
3782
+ case 'text': {
3783
+ const t = b.text ?? '';
3784
+ totalChars += t.length;
3785
+ // Fenced code blocks.
3786
+ const fences = t.split('```');
3787
+ for (let i = 1; i < fences.length; i += 2) externalChars += fences[i].length;
3788
+ // Bare link-drop lines (the URL is the payload).
3789
+ for (const line of t.split('\n')) {
3790
+ const trimmed = line.trim();
3791
+ if (/^https?:\/\/\S+$/.test(trimmed)) externalChars += trimmed.length;
3792
+ }
3793
+ break;
3794
+ }
3795
+ case 'tool_use': {
3796
+ const n = JSON.stringify(b.input ?? {}).length;
3797
+ totalChars += n;
3798
+ externalChars += n;
3799
+ break;
3800
+ }
3801
+ case 'tool_result': {
3802
+ const n =
3803
+ typeof b.content === 'string'
3804
+ ? b.content.length
3805
+ : JSON.stringify(b.content ?? '').length;
3806
+ totalChars += n;
3807
+ externalChars += n;
3808
+ break;
3809
+ }
3810
+ case 'image': {
3811
+ // Estimate parity with the renderer's flat image cost; the payload
3812
+ // lives in the file/CDN, so it is fully externalized.
3813
+ totalChars += 6400; // ≈1600 tokens × 4 chars
3814
+ externalChars += 6400;
3815
+ break;
3816
+ }
3817
+ default: {
3818
+ const t = (b as { text?: string }).text ?? '';
3819
+ totalChars += t.length;
3820
+ }
3821
+ }
3822
+ }
3823
+ if (totalChars <= 0) return 1;
3824
+ const externalized = Math.min(1, externalChars / totalChars);
3825
+ // Fully-externalized content bottoms out at 0.2 — cheap, never free
3826
+ // (hard protections, not salience, are what make content unfoldable).
3827
+ return Math.max(0.2, 1 - 0.8 * externalized);
3828
+ }
3829
+
2861
3830
  /**
2862
3831
  * Walk the summary tree to find the L_k ancestor of a message.
2863
3832
  * Returns null if no ancestor exists at that level (e.g., L_k not yet produced).
@@ -3020,6 +3989,76 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3020
3989
  selectedSummaries.push(...l1Selected);
3021
3990
  totalSummaryTokens += l1Used;
3022
3991
 
3992
+ // Phase 3b: coverage repair (bug 6.9). getAntiRedundantSummaries excluded
3993
+ // an L2 (or L3) when ALL of its children were in the CANDIDATE shown-set —
3994
+ // computed before budget selection. If the budget then dropped some of
3995
+ // those children (e.g. KnowledgeStrategy's research L1 cap), the covered
3996
+ // history appears at NEITHER level: a silent memory hole. Re-include any
3997
+ // excluded L2/L3 whose children did not all make the final selection.
3998
+ // Some overlap with the children that DID survive is accepted — coverage
3999
+ // beats perfect dedup here.
4000
+ //
4001
+ // Repairs are additionally bounded by a per-level ALLOWANCE (a fraction of
4002
+ // that level's budget), not just the overall context budget. The
4003
+ // excluded-with-partially-dropped-children state only arises from a store
4004
+ // damaged mid-merge (the crash window at compressChunkHierarchical, or the
4005
+ // legacy setMergedInto index-desync). On such a store MANY L2s can be in
4006
+ // this state at once; without a cap, re-including all of them at full size
4007
+ // would starve the recent window via Phase 4's newest-first eviction. When
4008
+ // repairs exceed the allowance we stop re-including and warn — a corrupted
4009
+ // store announces itself instead of silently trading recent messages for
4010
+ // redundant summaries.
4011
+ {
4012
+ // Allowance = a fraction of the level budget, with a floor tied to the
4013
+ // overall budget so a strategy that zeroes a level budget (e.g.
4014
+ // KnowledgeStrategy prioritising L1) can still repair a handful of
4015
+ // covering summaries, while a corrupted store with dozens of them stays
4016
+ // bounded well short of the recent window.
4017
+ const REPAIR_ALLOWANCE_FRACTION = 0.25;
4018
+ const REPAIR_FLOOR_FRACTION = 0.05;
4019
+ const repairFloor = maxTokens * REPAIR_FLOOR_FRACTION;
4020
+ const l2RepairAllowance = Math.max(l2Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
4021
+ const l3RepairAllowance = Math.max(l3Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
4022
+ const selectedIds = new Set(selectedSummaries.map(s => s.id));
4023
+ const shownL2Ids = new Set(shownL2.map(s => s.id));
4024
+ const shownL3Ids = new Set(shownL3.map(s => s.id));
4025
+ let l2RepairTokens = 0;
4026
+ let l3RepairTokens = 0;
4027
+ let l2RepairsSkipped = 0;
4028
+ let l3RepairsSkipped = 0;
4029
+ // L2s excluded by anti-redundancy: unmerged, not in shownL2.
4030
+ for (const s of this.summaries) {
4031
+ if (s.level !== 2 || s.mergedInto || shownL2Ids.has(s.id)) continue;
4032
+ if (s.sourceIds.every(id => selectedIds.has(id))) continue; // truly redundant
4033
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens)) continue;
4034
+ if (l2RepairTokens + s.tokens > l2RepairAllowance) { l2RepairsSkipped++; continue; }
4035
+ selectedSummaries.push(s);
4036
+ totalSummaryTokens += s.tokens;
4037
+ l2RepairTokens += s.tokens;
4038
+ selectedIds.add(s.id);
4039
+ }
4040
+ // L3s excluded by anti-redundancy — after L2 repair, so a repaired L2
4041
+ // counts as selected coverage for its parent L3.
4042
+ for (const s of this.summaries) {
4043
+ if (s.level !== 3 || s.mergedInto || shownL3Ids.has(s.id)) continue;
4044
+ if (s.sourceIds.every(id => selectedIds.has(id))) continue;
4045
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens)) continue;
4046
+ if (l3RepairTokens + s.tokens > l3RepairAllowance) { l3RepairsSkipped++; continue; }
4047
+ selectedSummaries.push(s);
4048
+ totalSummaryTokens += s.tokens;
4049
+ l3RepairTokens += s.tokens;
4050
+ selectedIds.add(s.id);
4051
+ }
4052
+ if (l2RepairsSkipped > 0 || l3RepairsSkipped > 0) {
4053
+ console.warn(
4054
+ `[AutobiographicalStrategy] coverage-repair allowance exceeded — ` +
4055
+ `skipped ${l2RepairsSkipped} L2 and ${l3RepairsSkipped} L3 re-inclusions ` +
4056
+ `(store likely corrupted mid-merge). Some covered history may render at ` +
4057
+ `no summary level this pass.`,
4058
+ );
4059
+ }
4060
+ }
4061
+
3023
4062
  // Emit summaries + pinned messages between head and recent windows.
3024
4063
  //
3025
4064
  // Default (positionedRecallPairs=true): one Q/A pair per summary,
@@ -3103,6 +4142,10 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3103
4142
  for (const item of items) {
3104
4143
  if (item.kind === 'summary') {
3105
4144
  const summary = item.summary;
4145
+ // Defensive: never emit a recall pair for an empty/bugged summary — an
4146
+ // empty assistant text block triggers a 400. (Production guards too,
4147
+ // but a legacy empty summary may already exist in the store.)
4148
+ if (!summary.content || !summary.content.trim()) continue;
3106
4149
  const headerText = this.buildRecallHeader(summary);
3107
4150
  const questionEntry: ContextEntry = {
3108
4151
  index: entries.length,
@@ -3208,6 +4251,9 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3208
4251
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
3209
4252
 
3210
4253
  this.trimOrphanedToolUse(entries);
4254
+ // Full pairing invariant over the final rendered context — catches the
4255
+ // mid-list orphans the trailing/leading trims can't (bug 6.7).
4256
+ this.enforceToolPairing(entries);
3211
4257
  this.pruneToolEntries(entries);
3212
4258
  // Strip stale images before committing stats so RenderStats.total reflects
3213
4259
  // the post-strip context (this path places no cache markers).
@@ -3520,27 +4566,83 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3520
4566
  }
3521
4567
 
3522
4568
  /**
3523
- * Rebuild chunk boundaries based on current messages.
4569
+ * Rebuild the chunk list: persisted records own the past; the running-sum
4570
+ * chunker only extends at the frontier, and a chunk is only ever created
4571
+ * once it CLOSES (reaches targetChunkTokens). The trailing partial chunk
4572
+ * is never created and never compressed — eager partial-tail compression
4573
+ * minted a new near-duplicate L1 per rebuild while the tail grew (the
4574
+ * prefix-generation families found fleet-wide in the 2026-07 audit).
3524
4575
  */
3525
4576
  protected rebuildChunks(store: MessageStoreView): void {
3526
- const messagesToChunk = this.getCompressibleMessages(store);
4577
+ this.chunks = [];
4578
+ this.compressionQueue = [];
3527
4579
 
3528
- // Preserve existing compressed chunks (legacy) and summary linkage (hierarchical)
3529
- const existingCompressed = new Map<string, Chunk>();
4580
+ // ---- 1. Materialize persisted records (they OWN their messages). ----
4581
+ const byId = new Map<string, StoredMessage>();
4582
+ for (const m of store.getAll()) byId.set(m.id, m);
4583
+
4584
+ const consumed = new Set<string>();
4585
+ let orphaned = 0;
4586
+ for (const rec of this.chunkRecords) {
4587
+ const msgs: StoredMessage[] = [];
4588
+ for (const id of rec.sourceIds) {
4589
+ const m = byId.get(id);
4590
+ if (m) msgs.push(m);
4591
+ }
4592
+ if (msgs.length === 0) { orphaned++; continue; }
4593
+ for (const m of msgs) consumed.add(m.id);
4594
+ const chunk: Chunk = {
4595
+ index: this.chunks.length,
4596
+ startIndex: -1, // record-derived; filtered-array indices are not meaningful
4597
+ endIndex: -1,
4598
+ messages: msgs,
4599
+ tokens: msgs.reduce((sum, m) => sum + (this.config.attachmentsIgnoreSize
4600
+ ? this.estimateTextOnlyTokens(m)
4601
+ : store.estimateTokens(m)), 0),
4602
+ compressed: rec.compressed,
4603
+ summaryId: rec.summaryId,
4604
+ phaseType: rec.phaseType,
4605
+ recordId: rec.id,
4606
+ };
4607
+ this.chunks.push(chunk);
4608
+ }
4609
+
4610
+ // ---- 2. Fail closed on the chain-break signature. ----
4611
+ // Most records resolving to zero live messages means message identity
4612
+ // has been rebuilt/renumbered underneath us. Chunking "fresh" ground now
4613
+ // would re-compress already-lived history into duplicate memories.
4614
+ // Halt ALL compression until an operator reconciles the store.
4615
+ if (this.chunkPersistenceEnabled && this.chunkRecords.length >= 3 &&
4616
+ orphaned / this.chunkRecords.length > 0.5) {
4617
+ this.chunkRecordsOrphaned = true;
4618
+ this.compressionQueue = [];
4619
+ if (!this._orphanWarned) {
4620
+ this._orphanWarned = true;
4621
+ console.error(
4622
+ `[autobiographical] FAIL-CLOSED: ${orphaned}/${this.chunkRecords.length} chunk ` +
4623
+ `records resolve to zero live messages (messages chain break / store ` +
4624
+ `reconciliation signature). Compression halted to prevent duplicate ` +
4625
+ `memory formation — reconcile the store before resuming.`,
4626
+ );
4627
+ }
4628
+ return;
4629
+ }
4630
+ this.chunkRecordsOrphaned = false;
4631
+
4632
+ // Queue uncompressed record-backed chunks (crash-recovery: record was
4633
+ // appended but the process died before its L1 landed).
3530
4634
  for (const chunk of this.chunks) {
3531
- if (chunk.compressed) {
3532
- const key = this.chunkKey(chunk);
3533
- existingCompressed.set(key, chunk);
4635
+ if (!chunk.compressed && !(chunk.recordId && this._overlapBlocked.has(chunk.recordId))) {
4636
+ this.compressionQueue.push(chunk.index);
3534
4637
  }
3535
4638
  }
3536
4639
 
3537
- // Rebuild chunks
3538
- this.chunks = [];
3539
- this.compressionQueue = [];
4640
+ // ---- 3. Chunk the frontier: compressible messages not owned by any record. ----
4641
+ const messagesToChunk = this.getCompressibleMessages(store)
4642
+ .filter(m => !consumed.has(m.id));
3540
4643
 
3541
4644
  let currentChunk: StoredMessage[] = [];
3542
4645
  let currentTokens = 0;
3543
- // Track start position in the filtered array for chunk boundary metadata
3544
4646
  let chunkFilteredStart = 0;
3545
4647
 
3546
4648
  for (let i = 0; i < messagesToChunk.length; i++) {
@@ -3570,19 +4672,27 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3570
4672
  }
3571
4673
 
3572
4674
  if (shouldClose) {
3573
- const chunk = this.createChunk(
3574
- this.chunks.length,
3575
- chunkFilteredStart,
3576
- i + 1,
3577
- currentChunk,
3578
- currentTokens,
3579
- existingCompressed
3580
- );
3581
- this.chunks.push(chunk);
3582
-
3583
- if (!chunk.compressed) {
3584
- this.compressionQueue.push(chunk.index);
4675
+ const chunk: Chunk = {
4676
+ index: this.chunks.length,
4677
+ startIndex: chunkFilteredStart,
4678
+ endIndex: i + 1,
4679
+ messages: [...currentChunk],
4680
+ tokens: currentTokens,
4681
+ compressed: false,
4682
+ };
4683
+ // Persist the boundary the moment it closes — from here on this
4684
+ // span is owned and never re-keyed by config drift or restarts.
4685
+ if (this.chunkPersistenceEnabled) {
4686
+ const record: ChunkRecord = {
4687
+ id: `c-${this.chunkIdCounter++}`,
4688
+ sourceIds: chunk.messages.map(m => m.id),
4689
+ compressed: false,
4690
+ };
4691
+ this.appendChunkRecord(record);
4692
+ chunk.recordId = record.id;
3585
4693
  }
4694
+ this.chunks.push(chunk);
4695
+ this.compressionQueue.push(chunk.index);
3586
4696
 
3587
4697
  currentChunk = [];
3588
4698
  currentTokens = 0;
@@ -3590,20 +4700,27 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3590
4700
  }
3591
4701
  }
3592
4702
 
3593
- if (currentChunk.length >= 4) {
3594
- const chunk = this.createChunk(
3595
- this.chunks.length,
3596
- chunkFilteredStart,
3597
- messagesToChunk.length,
3598
- currentChunk,
3599
- currentTokens,
3600
- existingCompressed
3601
- );
3602
- this.chunks.push(chunk);
3603
-
3604
- if (!chunk.compressed) {
3605
- this.compressionQueue.push(chunk.index);
3606
- }
4703
+ // NOTE: no trailing-partial chunk. An unclosed chunk is not a chunk —
4704
+ // it compresses only after the running sum closes it.
4705
+
4706
+ // ---- 4. L1 holdback: keep the newest X closed chunks out of the
4707
+ // speculative queue (default 1). The chunk at the live edge is the one
4708
+ // most likely to still be in motion (edits, tool-result landings, the
4709
+ // episode it belongs to still resolving); summarize it once a newer chunk
4710
+ // has closed behind it. The queue is rebuilt on every message, so a
4711
+ // held-back chunk is released automatically the moment it ages out of the
4712
+ // window. Demand overrides: a picker `produce` op (enqueueL1ForRange)
4713
+ // marks the chunk demanded, and demanded chunks are never held back —
4714
+ // when folding actually NEEDS the L1, production must not be blocked.
4715
+ const holdback = this.config.l1HoldbackChunks ?? 1;
4716
+ if (holdback > 0 && this.chunks.length > 0) {
4717
+ const cutoff = this.chunks.length - holdback;
4718
+ this.compressionQueue = this.compressionQueue.filter((idx) => {
4719
+ if (idx < cutoff) return true;
4720
+ const ch = this.chunks[idx];
4721
+ const lastId = ch?.messages[ch.messages.length - 1]?.id;
4722
+ return lastId !== undefined && this._demandedL1Chunks.has(lastId);
4723
+ });
3607
4724
  }
3608
4725
  }
3609
4726
 
@@ -3701,7 +4818,8 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3701
4818
  protected applyImageStripping(entries: ContextEntry[], store: MessageStoreView): void {
3702
4819
  const maxLive = this.config.maxLiveImages ?? 0; // 0 = unlimited count
3703
4820
  const depthTokens = this.config.imageStripDepthTokens ?? 0; // 0 = no depth strip
3704
- if (maxLive === 0 && depthTokens === 0) return; // policy disabled
4821
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4822
+ if (maxLive === 0 && depthTokens === 0 && maxLiveBytes === 0) return; // policy disabled
3705
4823
 
3706
4824
  const messages = store.getAll();
3707
4825
  const posById = new Map<string, number>();
@@ -3732,34 +4850,152 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3732
4850
  .sort((a, b) => b.pos - a.pos);
3733
4851
 
3734
4852
  let keptImages = 0;
4853
+ let keptImageBytes = 0;
3735
4854
  for (const { idx, pos } of ordered) {
3736
4855
  const entry = entries[idx];
3737
4856
  const tooDeep = depthTokens > 0 && (pos < 0 || pos < stripStart);
3738
4857
  const bucket = bucketAt(pos);
3739
4858
  entry.content = entry.content.map((block) => {
3740
4859
  if (block.type !== 'image') return block;
4860
+ const blockBytes = AutobiographicalStrategy.imageBlockBytes(block);
3741
4861
  const overCount = maxLive > 0 && keptImages >= maxLive;
3742
- if (tooDeep || overCount) {
3743
- // The renderer estimates an image at `tokenEstimate ?? 1600` (see
3744
- // message-store/context-log). Hand that back to the bucket, less the
3745
- // placeholder text that replaces it, so the stats match the output.
3746
- const reclaimed =
3747
- ((block as { tokenEstimate?: number }).tokenEstimate ?? 1600) - placeholderTokens;
3748
- if (this._rs && reclaimed > 0) this._rs[bucket].tokens -= reclaimed;
4862
+ const overBytes = maxLiveBytes > 0 && keptImageBytes + blockBytes > maxLiveBytes;
4863
+ if (tooDeep || overCount || overBytes) {
4864
+ // Stats-neutral (2026-07-12): every budgeting site now tallies at
4865
+ // POST-STRIP prices (see postStripEstimates), so the bucket never
4866
+ // charged this image at full weight — reclaiming here would
4867
+ // double-decrement. The strip pass only swaps the block.
4868
+ void bucket;
4869
+ void placeholderTokens;
3749
4870
  return { type: 'text', text: AutobiographicalStrategy.IMAGE_PLACEHOLDER } as ContentBlock;
3750
4871
  }
3751
4872
  keptImages++;
4873
+ keptImageBytes += blockBytes;
3752
4874
  return block;
3753
4875
  });
3754
4876
  }
3755
4877
  }
3756
4878
 
4879
+ /**
4880
+ * Post-strip token estimate per message index (2026-07-12 tail-starvation
4881
+ * fix). Mirrors `applyImageStripping`: an image beyond the `maxLiveImages`
4882
+ * newest (counted newest-first) or deeper than `imageStripDepthTokens` of
4883
+ * raw estimate from the live end renders as a placeholder — so every place
4884
+ * that BUDGETS messages (recent-window walk-back, head/tail sums, middle
4885
+ * chunk sizes) must cost it as one. Pricing stripped images at their full
4886
+ * estimate collapsed an image-dense tail to a fraction of its configured
4887
+ * size (42k rendered of a 120k window), and pricing them post-strip in the
4888
+ * walk-back alone made the picker's raw-priced tail overflow the budget
4889
+ * (318k) — the estimate must be consistent EVERYWHERE.
4890
+ */
4891
+ protected postStripEstimates(store: MessageStoreView): number[] {
4892
+ const messages = store.getAll();
4893
+ const out = new Array<number>(messages.length);
4894
+ const stripDepth = this.config.imageStripDepthTokens ?? 0;
4895
+ const maxLive = this.config.maxLiveImages ?? 0;
4896
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4897
+ const stripActive = stripDepth > 0 || maxLive > 0 || maxLiveBytes > 0;
4898
+ const placeholderTokens = Math.ceil(AutobiographicalStrategy.IMAGE_PLACEHOLDER.length / 4);
4899
+ let liveImagesSeen = 0;
4900
+ let liveImageBytes = 0;
4901
+ let rawDepth = 0; // raw-estimate depth from the newest message (mirrors getImageStripStart)
4902
+ for (let i = messages.length - 1; i >= 0; i--) {
4903
+ const raw = store.estimateTokens(messages[i]);
4904
+ let est = raw;
4905
+ if (stripActive) {
4906
+ for (const b of messages[i].content) {
4907
+ if (b.type !== 'image') continue;
4908
+ const bytes = AutobiographicalStrategy.imageBlockBytes(b);
4909
+ const beyondDepth = stripDepth > 0 && rawDepth > stripDepth;
4910
+ const beyondCount = maxLive > 0 && liveImagesSeen >= maxLive;
4911
+ const beyondBytes = maxLiveBytes > 0 && liveImageBytes + bytes > maxLiveBytes;
4912
+ if (beyondDepth || beyondCount || beyondBytes) {
4913
+ const imgEst = (b as { tokenEstimate?: number }).tokenEstimate ?? 1600;
4914
+ est -= Math.max(0, imgEst - placeholderTokens);
4915
+ } else {
4916
+ liveImagesSeen++;
4917
+ liveImageBytes += bytes;
4918
+ }
4919
+ }
4920
+ }
4921
+ rawDepth += raw;
4922
+ out[i] = est;
4923
+ }
4924
+ return out;
4925
+ }
4926
+
4927
+ /** Byte wall default: 20MB of base64 (API total-request cap is 32MB). */
4928
+ protected static readonly DEFAULT_MAX_LIVE_IMAGE_BYTES = 20 * 1024 * 1024;
4929
+
4930
+ /** Compression prompts carry head + recall frontier + raw chunk alongside
4931
+ * their images, so they get a tighter image budget than the live window. */
4932
+ protected static readonly DEFAULT_MAX_COMPRESSION_IMAGE_BYTES = 12 * 1024 * 1024;
4933
+
4934
+ /** Base64 payload size of an image block (0 for non-base64 sources). */
4935
+ protected static imageBlockBytes(b: unknown): number {
4936
+ const src = (b as { source?: { data?: string } }).source;
4937
+ return typeof src?.data === 'string' ? src.data.length : 0;
4938
+ }
4939
+
4940
+ /**
4941
+ * Cap inline image bytes in a COMPRESSION prompt (2026-07-12). The main
4942
+ * window enforces `maxLiveImageBytes` through the strip policy; the
4943
+ * summarizer's raw-chunk replay had no such wall and leaned on membrane's
4944
+ * byte shed instead — which is a transport backstop, not a policy owner
4945
+ * (it fired at 27MB on a live merge). Keep images newest-first within the
4946
+ * budget; older ones become the same loud placeholder the window uses, so
4947
+ * the summarizer is told plainly that it is not seeing them.
4948
+ * Returns the number of images replaced.
4949
+ */
4950
+ protected capCompressionImageBytes(
4951
+ messages: Array<{ content: ContentBlock[] }>,
4952
+ capBytes: number,
4953
+ ): number {
4954
+ if (capBytes <= 0) return 0;
4955
+ let kept = 0;
4956
+ let dropped = 0;
4957
+ // Recurse into tool_result content: an agent that drives a shell/plotter/
4958
+ // browser carries most of its image bytes NESTED in tool results, not as
4959
+ // top-level blocks. Capping only the top level left those untouched and
4960
+ // membrane's transport shed kept firing at 27MB (2026-07-12).
4961
+ const capBlocks = (blocks: ContentBlock[]): ContentBlock[] =>
4962
+ blocks.map((b) => {
4963
+ if (b.type === 'image') {
4964
+ const bytes = AutobiographicalStrategy.imageBlockBytes(b);
4965
+ if (kept + bytes <= capBytes) {
4966
+ kept += bytes;
4967
+ return b;
4968
+ }
4969
+ dropped++;
4970
+ return { type: 'text', text: AutobiographicalStrategy.IMAGE_PLACEHOLDER } as ContentBlock;
4971
+ }
4972
+ const nested = (b as { type: string; content?: unknown }).content;
4973
+ if (b.type === 'tool_result' && Array.isArray(nested)) {
4974
+ return { ...b, content: capBlocks(nested as ContentBlock[]) } as ContentBlock;
4975
+ }
4976
+ return b;
4977
+ });
4978
+ for (let i = messages.length - 1; i >= 0; i--) {
4979
+ const m = messages[i];
4980
+ if (!Array.isArray(m.content)) continue;
4981
+ m.content = capBlocks(m.content);
4982
+ }
4983
+ if (dropped > 0) {
4984
+ console.error(
4985
+ `[autobiographical] compression prompt: replaced ${dropped} older image(s) with placeholders ` +
4986
+ `to stay under the ${Math.round(capBytes / 1e6)}MB image-byte budget (kept ${Math.round(kept / 1e6)}MB, newest-first)`,
4987
+ );
4988
+ }
4989
+ return dropped;
4990
+ }
4991
+
3757
4992
  protected getRecentWindowStart(store: MessageStoreView): number {
3758
4993
  const messages = store.getAll();
4994
+ const pse = this.postStripEstimates(store);
3759
4995
  let tokens = 0;
3760
4996
 
3761
4997
  for (let i = messages.length - 1; i >= 0; i--) {
3762
- tokens += store.estimateTokens(messages[i]);
4998
+ tokens += pse[i];
3763
4999
  if (tokens > this.config.recentWindowTokens) {
3764
5000
  let boundary = i + 1;
3765
5001
  // Don't split a tool_use/tool_result pair: if the message at the boundary
@@ -3847,6 +5083,180 @@ export class AutobiographicalStrategy implements ResettableStrategy {
3847
5083
  }
3848
5084
  }
3849
5085
 
5086
+ /** Placeholder body for a stub tool_result inserted by enforceToolPairing. */
5087
+ private static readonly STUB_TOOL_RESULT_TEXT =
5088
+ '[tool result unavailable — omitted during context compression]';
5089
+
5090
+ /**
5091
+ * Final post-selection tool-pairing validator (bug 6.7).
5092
+ *
5093
+ * The Anthropic API requires every `tool_use` block to be answered by a
5094
+ * matching `tool_result` in the immediately-following message, and every
5095
+ * `tool_result` to answer a `tool_use` in the immediately-preceding
5096
+ * message. Selection can violate this mid-list in ways the trailing
5097
+ * (`trimOrphanedToolUse`) and leading orphan trims don't catch:
5098
+ *
5099
+ * - a budget `break` cutting between a raw pin pair's two messages;
5100
+ * - the uncompressed-chunk fallback emitting a raw tool_result whose
5101
+ * tool_use chunk already compressed (or vice versa);
5102
+ * - a recall pair or pin interleaving between a tool_use and its result.
5103
+ *
5104
+ * Repair policy prefers preserving content over dropping:
5105
+ *
5106
+ * - a tool_use whose result is missing from the next entry first triggers
5107
+ * a short look-ahead: if the genuine (displaced) result is a few entries
5108
+ * down it is MOVED up into position (see
5109
+ * {@link relocateOrDropMissingResults}); only when no real result exists
5110
+ * is a STUB tool_result emitted. Either way the result block is merged
5111
+ * into the next entry when that entry already carries results for this
5112
+ * cycle, or inserted as a new user entry;
5113
+ * - a tool_result whose tool_use is not in the immediately-preceding
5114
+ * entry (and was not relocated) is dropped — there is no safe way to
5115
+ * stub a tool_use, so the result's information content survives only if
5116
+ * its use is adjacent; an entry left empty is replaced with a
5117
+ * placeholder text block.
5118
+ *
5119
+ * Runs as a structural pass over the rendered context in BOTH render
5120
+ * paths — `selectHierarchical` (downstream of `selectL1Summaries`, so
5121
+ * subclass overrides like KnowledgeStrategy are covered) and
5122
+ * `selectAdaptive` (the path FKM defaults onto). It's a no-op on
5123
+ * already-valid output.
5124
+ */
5125
+ protected enforceToolPairing(entries: ContextEntry[]): void {
5126
+ let prevUseIds = new Set<string>();
5127
+
5128
+ for (let i = 0; i < entries.length; i++) {
5129
+ const entry = entries[i];
5130
+
5131
+ // --- Rule A: every tool_result must answer a tool_use in the
5132
+ // immediately-preceding entry (and only once). Drop orphans/dupes. ---
5133
+ if (entry.content.some(b => b.type === 'tool_result')) {
5134
+ const seen = new Set<string>();
5135
+ const filtered = entry.content.filter(b => {
5136
+ if (b.type !== 'tool_result') return true;
5137
+ if (!prevUseIds.has(b.toolUseId) || seen.has(b.toolUseId)) return false;
5138
+ seen.add(b.toolUseId);
5139
+ return true;
5140
+ });
5141
+ if (filtered.length !== entry.content.length) {
5142
+ entry.content = filtered.length > 0
5143
+ ? filtered
5144
+ : [{ type: 'text', text: '[tool call omitted]' }];
5145
+ }
5146
+ }
5147
+
5148
+ // --- Rule B: every tool_use in the PREVIOUS entry must be answered
5149
+ // by this entry. Stub any that aren't. ---
5150
+ if (prevUseIds.size > 0) {
5151
+ const answered = new Set<string>();
5152
+ for (const b of entry.content) {
5153
+ if (b.type === 'tool_result') answered.add(b.toolUseId);
5154
+ }
5155
+ const missing = [...prevUseIds].filter(id => !answered.has(id));
5156
+ if (missing.length > 0) {
5157
+ // Look-ahead relocation: the genuine result for a "missing" id is
5158
+ // often sitting a few entries down, displaced by an interleaved
5159
+ // recall pair / pin (it will otherwise be dropped as an orphan by
5160
+ // Rule A when we reach it). Move the real block up into position and
5161
+ // only stub the ids for which no real result exists — so tool output
5162
+ // is preserved, not silently replaced by a placeholder.
5163
+ const results = this.relocateOrDropMissingResults(entries, i, missing);
5164
+ if (answered.size > 0) {
5165
+ // This entry already carries results for the cycle — prepend the
5166
+ // relocated/stub results so all results for the preceding tool_use
5167
+ // sit together (the API wants tool_result blocks at the head of
5168
+ // the message).
5169
+ entry.content = [...results, ...entry.content];
5170
+ } else {
5171
+ // Not a results entry at all — insert a synthetic user entry
5172
+ // between the tool_use entry and this one.
5173
+ entries.splice(i, 0, {
5174
+ index: i,
5175
+ participant: 'user',
5176
+ sourceRelation: 'derived',
5177
+ content: results,
5178
+ });
5179
+ // The stub entry (no tool_use blocks) is now at index i; the
5180
+ // current entry moved to i+1 and is re-processed next iteration
5181
+ // with an empty prevUseIds (its orphan results, if any, were
5182
+ // already filtered above and none survived — Rule A matched
5183
+ // against the same prevUseIds and answered.size === 0).
5184
+ prevUseIds = new Set();
5185
+ continue;
5186
+ }
5187
+ }
5188
+ }
5189
+
5190
+ prevUseIds = new Set();
5191
+ for (const b of entry.content) {
5192
+ if (b.type === 'tool_use') prevUseIds.add(b.id);
5193
+ }
5194
+ }
5195
+
5196
+ // Tail: an entry that mixes tool_use with other blocks can survive
5197
+ // trimOrphanedToolUse (which only pops pure use-without-result tails).
5198
+ if (prevUseIds.size > 0) {
5199
+ entries.push({
5200
+ index: entries.length,
5201
+ participant: 'user',
5202
+ sourceRelation: 'derived',
5203
+ content: [...prevUseIds].map(id => ({
5204
+ type: 'tool_result' as const,
5205
+ toolUseId: id,
5206
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
5207
+ })),
5208
+ });
5209
+ }
5210
+
5211
+ // Reindex after any splices/appends.
5212
+ for (let i = 0; i < entries.length; i++) entries[i].index = i;
5213
+ }
5214
+
5215
+ /**
5216
+ * For each `missing` tool_use id (a use in the preceding entry with no
5217
+ * adjacent result), return the result block to place next to it:
5218
+ *
5219
+ * - if the genuine result exists within a short look-ahead window after
5220
+ * `afterIndex`, MOVE it up into position (removing it from its source
5221
+ * entry — replacing a now-empty entry with a placeholder) so the real
5222
+ * tool output survives the repair;
5223
+ * - otherwise emit a stub.
5224
+ *
5225
+ * tool_use ids are unique, so the single result carrying a given id is
5226
+ * unambiguously the answer to that use — relocating it cannot break any
5227
+ * other pairing. Results for `missing` ids never sit at or before
5228
+ * `afterIndex` (that entry's results are already matched), so the search
5229
+ * starts at `afterIndex + 1`.
5230
+ */
5231
+ private relocateOrDropMissingResults(
5232
+ entries: ContextEntry[],
5233
+ afterIndex: number,
5234
+ missing: string[],
5235
+ ): ContentBlock[] {
5236
+ const LOOKAHEAD = 6;
5237
+ const end = Math.min(entries.length, afterIndex + 1 + LOOKAHEAD);
5238
+ return missing.map(id => {
5239
+ for (let j = afterIndex + 1; j < end; j++) {
5240
+ const src = entries[j];
5241
+ const bi = src.content.findIndex(
5242
+ b => b.type === 'tool_result' && b.toolUseId === id,
5243
+ );
5244
+ if (bi === -1) continue;
5245
+ const real = src.content[bi];
5246
+ const rest = src.content.filter((_, k) => k !== bi);
5247
+ src.content = rest.length > 0
5248
+ ? rest
5249
+ : [{ type: 'text', text: '[tool call omitted]' }];
5250
+ return real;
5251
+ }
5252
+ return {
5253
+ type: 'tool_result',
5254
+ toolUseId: id,
5255
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
5256
+ } as ContentBlock;
5257
+ });
5258
+ }
5259
+
3850
5260
  /**
3851
5261
  * Prune tool_use / tool_result blocks in-place:
3852
5262
  * 1. Truncate `tool_use.input` blocks whose serialized JSON exceeds