@animalabs/context-manager 0.5.5 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/src/adaptive/kv-control.d.ts +69 -32
  2. package/dist/src/adaptive/kv-control.d.ts.map +1 -1
  3. package/dist/src/adaptive/kv-control.js +471 -172
  4. package/dist/src/adaptive/kv-control.js.map +1 -1
  5. package/dist/src/adaptive/picker.d.ts +8 -0
  6. package/dist/src/adaptive/picker.d.ts.map +1 -1
  7. package/dist/src/adaptive/picker.js.map +1 -1
  8. package/dist/src/adaptive/strategies/kv-stable.d.ts +12 -4
  9. package/dist/src/adaptive/strategies/kv-stable.d.ts.map +1 -1
  10. package/dist/src/adaptive/strategies/kv-stable.js +14 -5
  11. package/dist/src/adaptive/strategies/kv-stable.js.map +1 -1
  12. package/dist/src/context-log.d.ts.map +1 -1
  13. package/dist/src/context-log.js +11 -0
  14. package/dist/src/context-log.js.map +1 -1
  15. package/dist/src/context-manager.d.ts +28 -1
  16. package/dist/src/context-manager.d.ts.map +1 -1
  17. package/dist/src/context-manager.js +42 -0
  18. package/dist/src/context-manager.js.map +1 -1
  19. package/dist/src/index.d.ts +3 -2
  20. package/dist/src/index.d.ts.map +1 -1
  21. package/dist/src/index.js +1 -0
  22. package/dist/src/index.js.map +1 -1
  23. package/dist/src/message-store.d.ts +60 -0
  24. package/dist/src/message-store.d.ts.map +1 -1
  25. package/dist/src/message-store.js +165 -10
  26. package/dist/src/message-store.js.map +1 -1
  27. package/dist/src/strategies/autobiographical.d.ts +223 -2
  28. package/dist/src/strategies/autobiographical.d.ts.map +1 -1
  29. package/dist/src/strategies/autobiographical.js +1487 -142
  30. package/dist/src/strategies/autobiographical.js.map +1 -1
  31. package/dist/src/strategies/index.d.ts +1 -1
  32. package/dist/src/strategies/index.d.ts.map +1 -1
  33. package/dist/src/strategies/index.js.map +1 -1
  34. package/dist/src/strategies/knowledge.d.ts +8 -0
  35. package/dist/src/strategies/knowledge.d.ts.map +1 -1
  36. package/dist/src/strategies/knowledge.js +23 -1
  37. package/dist/src/strategies/knowledge.js.map +1 -1
  38. package/dist/src/types/strategy.d.ts +89 -5
  39. package/dist/src/types/strategy.d.ts.map +1 -1
  40. package/dist/src/types/strategy.js.map +1 -1
  41. package/dist/test/adaptive/doc-plus-chat.test.js +7 -3
  42. package/dist/test/adaptive/doc-plus-chat.test.js.map +1 -1
  43. package/dist/test/adaptive/hard-fail.test.js +103 -0
  44. package/dist/test/adaptive/hard-fail.test.js.map +1 -1
  45. package/dist/test/adaptive/ingestion.test.js +3 -1
  46. package/dist/test/adaptive/ingestion.test.js.map +1 -1
  47. package/dist/test/adaptive/kv-control.test.js +17 -11
  48. package/dist/test/adaptive/kv-control.test.js.map +1 -1
  49. package/dist/test/adaptive/single-path-solve.test.d.ts +16 -0
  50. package/dist/test/adaptive/single-path-solve.test.d.ts.map +1 -0
  51. package/dist/test/adaptive/single-path-solve.test.js +204 -0
  52. package/dist/test/adaptive/single-path-solve.test.js.map +1 -0
  53. package/dist/test/anti-redundancy-budget.test.d.ts +17 -0
  54. package/dist/test/anti-redundancy-budget.test.d.ts.map +1 -0
  55. package/dist/test/anti-redundancy-budget.test.js +197 -0
  56. package/dist/test/anti-redundancy-budget.test.js.map +1 -0
  57. package/dist/test/chunk-persistence.test.d.ts +29 -0
  58. package/dist/test/chunk-persistence.test.d.ts.map +1 -0
  59. package/dist/test/chunk-persistence.test.js +290 -0
  60. package/dist/test/chunk-persistence.test.js.map +1 -0
  61. package/dist/test/compression-dedup.test.d.ts +20 -0
  62. package/dist/test/compression-dedup.test.d.ts.map +1 -0
  63. package/dist/test/compression-dedup.test.js +92 -0
  64. package/dist/test/compression-dedup.test.js.map +1 -0
  65. package/dist/test/compression-shape-invariants.test.js +40 -3
  66. package/dist/test/compression-shape-invariants.test.js.map +1 -1
  67. package/dist/test/head-window-order.test.d.ts +19 -0
  68. package/dist/test/head-window-order.test.d.ts.map +1 -0
  69. package/dist/test/head-window-order.test.js +159 -0
  70. package/dist/test/head-window-order.test.js.map +1 -0
  71. package/dist/test/image-strip-render-stats.test.js +25 -3
  72. package/dist/test/image-strip-render-stats.test.js.map +1 -1
  73. package/dist/test/knowledge-tool-boundary.test.d.ts +14 -0
  74. package/dist/test/knowledge-tool-boundary.test.d.ts.map +1 -0
  75. package/dist/test/knowledge-tool-boundary.test.js +93 -0
  76. package/dist/test/knowledge-tool-boundary.test.js.map +1 -0
  77. package/dist/test/l1-holdback.test.d.ts +15 -0
  78. package/dist/test/l1-holdback.test.d.ts.map +1 -0
  79. package/dist/test/l1-holdback.test.js +119 -0
  80. package/dist/test/l1-holdback.test.js.map +1 -0
  81. package/dist/test/merge-contiguity.test.d.ts +11 -0
  82. package/dist/test/merge-contiguity.test.d.ts.map +1 -0
  83. package/dist/test/merge-contiguity.test.js +82 -0
  84. package/dist/test/merge-contiguity.test.js.map +1 -0
  85. package/dist/test/message-store-window.test.d.ts +13 -0
  86. package/dist/test/message-store-window.test.d.ts.map +1 -0
  87. package/dist/test/message-store-window.test.js +177 -0
  88. package/dist/test/message-store-window.test.js.map +1 -0
  89. package/dist/test/speculation-cap.test.js +4 -0
  90. package/dist/test/speculation-cap.test.js.map +1 -1
  91. package/dist/test/strategy-persistence.test.js +41 -0
  92. package/dist/test/strategy-persistence.test.js.map +1 -1
  93. package/dist/test/summary-log-consistency.test.d.ts +17 -0
  94. package/dist/test/summary-log-consistency.test.d.ts.map +1 -0
  95. package/dist/test/summary-log-consistency.test.js +102 -0
  96. package/dist/test/summary-log-consistency.test.js.map +1 -0
  97. package/dist/test/tool-pairing-invariant.test.d.ts +15 -0
  98. package/dist/test/tool-pairing-invariant.test.d.ts.map +1 -0
  99. package/dist/test/tool-pairing-invariant.test.js +225 -0
  100. package/dist/test/tool-pairing-invariant.test.js.map +1 -0
  101. package/dist/tsconfig.tsbuildinfo +1 -1
  102. package/package.json +11 -3
  103. package/src/adaptive/kv-control.ts +509 -184
  104. package/src/adaptive/picker.ts +8 -0
  105. package/src/adaptive/strategies/kv-stable.ts +22 -10
  106. package/src/context-log.ts +11 -0
  107. package/src/context-manager.ts +48 -2
  108. package/src/index.ts +3 -2
  109. package/src/message-store.ts +202 -9
  110. package/src/strategies/autobiographical.ts +1576 -166
  111. package/src/strategies/index.ts +1 -1
  112. package/src/strategies/knowledge.ts +25 -1
  113. package/src/types/strategy.ts +97 -5
@@ -54,7 +54,12 @@ function formatInstruction(targetTokens) {
54
54
  'Preserve concrete details — file paths, exact values, decisions, ' +
55
55
  `unresolved questions, the user\'s active asks. Target ~${targetTokens} ` +
56
56
  'tokens. Output only the memory body — no preamble, no section headers ' +
57
- 'unless they help preservation, no meta-commentary about summarizing.');
57
+ 'unless they help preservation, no meta-commentary about summarizing. ' +
58
+ 'Memorize only what actually happened in that slice: if it holds little ' +
59
+ 'beyond routine system traffic (heartbeats, empty turns, failure ' +
60
+ 'notices), a short memory saying so is correct — do not pad it by ' +
61
+ 're-narrating events you already remember from earlier as if they had ' +
62
+ 'just happened again.');
58
63
  }
59
64
  /**
60
65
  * Compression instruction for chunks that are part of a substantially larger
@@ -165,6 +170,39 @@ function normalizePinLevels(opts) {
165
170
  const maxLevel = clean(opts?.maxLevel);
166
171
  return maxLevel !== undefined ? { maxLevel } : {};
167
172
  }
173
+ /**
174
+ * Drop empty text blocks (`{type:'text', text:''}` or whitespace-only). The
175
+ * Anthropic API rejects them with 400 "text content blocks must be non-empty",
176
+ * which — thrown inside the speculative-compression drain — halts ALL
177
+ * compression. Non-text blocks (tool_use/tool_result/image) pass through
178
+ * unchanged, so tool pairing is preserved; callers drop any message left with
179
+ * an empty content array.
180
+ */
181
+ function stripEmptyTextBlocks(content) {
182
+ return content.filter((b) => {
183
+ if (b.type !== 'text')
184
+ return true;
185
+ const text = b.text;
186
+ return typeof text === 'string' && text.trim().length > 0;
187
+ });
188
+ }
189
+ /**
190
+ * Strip `thinking` / `redacted_thinking` blocks from compression INPUT.
191
+ *
192
+ * The summarizer must never be handed the agent's own reasoning:
193
+ * (a) signed thinking is only valid verbatim in the turn that produced it —
194
+ * replaying it into a rewritten summarize request corrupts the signature;
195
+ * (b) asking the model to summarize its own reasoning reads as reproducing /
196
+ * duplicating model output → `reasoning_extraction` refusal, which returns
197
+ * empty (→ "empty L1 summary, chunk left raw") or, worse, produces a
198
+ * summary that reproduces the reasoning as text — which then trips the
199
+ * SAME classifier on the MAIN thread once that summary is rendered.
200
+ * Thinking is scratch work, not history (the same rationale already applied to
201
+ * the summarizer's OUTPUT). Drop it from the input too.
202
+ */
203
+ function stripThinkingBlocks(content) {
204
+ return content.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking');
205
+ }
168
206
  /**
169
207
  * Autobiographical chunking strategy.
170
208
  * Compresses old conversation chunks into summaries in the model's own words.
@@ -190,6 +228,30 @@ export class AutobiographicalStrategy {
190
228
  * unchanged), halting the drain with work still queued.
191
229
  */
192
230
  _drainProgress = 0;
231
+ /**
232
+ * Chunks (keyed by their LAST message id — chunk membership is immutable
233
+ * once closed) whose L1 was explicitly demanded by a picker `produce` op.
234
+ * Demanded chunks bypass the `l1HoldbackChunks` window in `rebuildChunks`:
235
+ * speculation waits, demand doesn't. Entries become inert once the chunk
236
+ * compresses (compressed chunks are never re-queued), so no cleanup needed.
237
+ */
238
+ _demandedL1Chunks = new Set();
239
+ /**
240
+ * In-memory mirror of the persisted chunk records (autobio:chunks slot).
241
+ * Loaded in `loadPersistedState`, appended to when a chunk closes,
242
+ * updated by ID when its L1 lands.
243
+ */
244
+ chunkRecords = [];
245
+ chunkIdCounter = 0;
246
+ /**
247
+ * Fail-closed latch: set when most persisted records resolve to zero live
248
+ * messages (the messages-chain-break signature). While set, NO compression
249
+ * runs — duplicate memories are strictly worse than delayed compression.
250
+ */
251
+ chunkRecordsOrphaned = false;
252
+ _orphanWarned = false;
253
+ /** Record ids whose compression was refused by the L1 overlap guard. */
254
+ _overlapBlocked = new Set();
193
255
  // Hierarchical state
194
256
  summaries = [];
195
257
  summaryIdCounter = 0;
@@ -204,11 +266,13 @@ export class AutobiographicalStrategy {
204
266
  /** Namespace for state-id scoping. Set in `initialize()`. */
205
267
  ns = '';
206
268
  get summariesStateId() { return `${this.ns}/autobio:summaries`; }
269
+ get chunksStateId() { return `${this.ns}/autobio:chunks`; }
207
270
  get counterStateId() { return `${this.ns}/autobio:counter`; }
208
271
  get mergeQueueStateId() { return `${this.ns}/autobio:mergeQueue`; }
209
272
  get pinsStateId() { return `${this.ns}/autobio:pins`; }
210
273
  get resolutionsStateId() { return `${this.ns}/autobio:resolutions`; }
211
274
  get locksStateId() { return `${this.ns}/autobio:locks`; }
275
+ get calibrationStateId() { return `${this.ns}/autobio:calibration`; }
212
276
  /** Protected ranges (pins + documents). Loaded from chronicle in initialize. */
213
277
  pins = [];
214
278
  /** Monotonically increasing counter for pin ids. Persisted as part of the pins snapshot. */
@@ -345,7 +409,120 @@ export class AutobiographicalStrategy {
345
409
  break;
346
410
  }
347
411
  }
412
+ // Legacy stores (pre chunk-persistence) have L1 summaries but no chunk
413
+ // records — synthesize records from L1 sourceIds before the first
414
+ // rebuild, so covered ground is owned and never re-compressed.
415
+ this.migrateChunkRecords(ctx.messageStore);
348
416
  this.rebuildChunks(ctx.messageStore);
417
+ // Kick the merge ladder for pre-existing unmerged summaries. Normally a
418
+ // compression/merge completion does this, but a store that boots with a
419
+ // backlog above threshold and an empty queue (e.g. after a pyramid
420
+ // repair pruned duplicates and un-merged survivors) would otherwise
421
+ // never start consolidating. Idempotent: already-queued/merged sources
422
+ // are skipped.
423
+ if (this.config.hierarchical && !this.chunkRecordsOrphaned) {
424
+ this.checkMergeThreshold();
425
+ }
426
+ }
427
+ /**
428
+ * Whether chunk boundaries are persisted to the `autobio:chunks` slot and
429
+ * own the past. Subclasses with their own chunking (KnowledgeStrategy's
430
+ * semantic phases) opt out and keep the legacy recompute-every-rebuild
431
+ * behavior.
432
+ */
433
+ get chunkPersistenceEnabled() {
434
+ return true;
435
+ }
436
+ /**
437
+ * One-time lazy migration: a store with L1 summaries but an empty chunks
438
+ * slot predates chunk persistence. Each L1's sourceIds ARE the historical
439
+ * chunk boundary — synthesize a compressed record per L1, in message
440
+ * order. Stale generations from the old partial-tail compression bug
441
+ * (an L1 whose messages are ALL already covered by records synthesized so
442
+ * far — prefix families, same-range duplicates) get NO record; coverage is
443
+ * what blocks re-compression, and the repair tooling prunes their content
444
+ * separately.
445
+ */
446
+ migrateChunkRecords(store) {
447
+ if (!this.chunkPersistenceEnabled || !this.store)
448
+ return;
449
+ if (this.chunkRecords.length > 0)
450
+ return;
451
+ const l1s = this.summaries.filter(s => s.level === 1 && Array.isArray(s.sourceIds) && s.sourceIds.length > 0);
452
+ if (l1s.length === 0)
453
+ return;
454
+ const msgIndex = new Map();
455
+ store.getAll().forEach((m, i) => msgIndex.set(m.id, i));
456
+ // Sort by (start position asc, span length desc) so at each starting
457
+ // point the LONGEST generation claims the ground first.
458
+ const sorted = [...l1s].sort((a, b) => {
459
+ const sa = msgIndex.get(a.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
460
+ const sb = msgIndex.get(b.sourceIds[0]) ?? Number.MAX_SAFE_INTEGER;
461
+ return sa - sb || b.sourceIds.length - a.sourceIds.length;
462
+ });
463
+ const covered = new Set();
464
+ let skippedStale = 0;
465
+ let skippedGhost = 0;
466
+ for (const s of sorted) {
467
+ // An L1 none of whose messages exist anymore can't own live ground.
468
+ if (!s.sourceIds.some(id => msgIndex.has(id))) {
469
+ skippedGhost++;
470
+ continue;
471
+ }
472
+ // Fully-covered = stale generation / contained duplicate.
473
+ if (s.sourceIds.every(id => covered.has(id))) {
474
+ skippedStale++;
475
+ continue;
476
+ }
477
+ for (const id of s.sourceIds)
478
+ covered.add(id);
479
+ this.appendChunkRecord({
480
+ id: `c-${this.chunkIdCounter++}`,
481
+ sourceIds: [...s.sourceIds],
482
+ compressed: true,
483
+ summaryId: s.id,
484
+ ...(s.phaseType ? { phaseType: s.phaseType } : {}),
485
+ });
486
+ }
487
+ console.warn(`[autobiographical] chunk-persistence migration: ${l1s.length} L1s → ` +
488
+ `${this.chunkRecords.length} records (${skippedStale} stale generations, ` +
489
+ `${skippedGhost} fully-orphaned L1s skipped)`);
490
+ }
491
+ /** Append a record to the chunks slot + in-memory mirror. */
492
+ appendChunkRecord(record) {
493
+ this.chunkRecords.push(record);
494
+ this.store?.appendToStateJson(this.chunksStateId, record);
495
+ }
496
+ /**
497
+ * Mark a chunk's record compressed, linking its L1. Resolves the log slot
498
+ * by ID against the persisted array — never by in-memory index (see
499
+ * setMergedInto for the clobber this avoids).
500
+ */
501
+ markChunkRecordCompressed(recordId, summaryId) {
502
+ if (!this.chunkPersistenceEnabled || !recordId)
503
+ return;
504
+ const rec = this.chunkRecords.find(r => r.id === recordId);
505
+ if (!rec)
506
+ return;
507
+ rec.compressed = true;
508
+ rec.summaryId = summaryId;
509
+ if (!this.store)
510
+ return;
511
+ const stored = this.store.getStateJson(this.chunksStateId);
512
+ if (!Array.isArray(stored))
513
+ return;
514
+ const payload = Buffer.from(JSON.stringify(rec));
515
+ let found = false;
516
+ for (let i = 0; i < stored.length; i++) {
517
+ const item = stored[i];
518
+ if (item && item.id === recordId) {
519
+ this.store.editStateItem(this.chunksStateId, i, payload);
520
+ found = true;
521
+ }
522
+ }
523
+ if (!found) {
524
+ console.warn(`[autobiographical] markChunkRecordCompressed: ${recordId} not found in persisted chunk log`);
525
+ }
349
526
  }
350
527
  /**
351
528
  * Register the three Chronicle state slots this strategy uses.
@@ -364,6 +541,17 @@ export class AutobiographicalStrategy {
364
541
  });
365
542
  }
366
543
  catch { /* already registered */ }
544
+ if (this.chunkPersistenceEnabled) {
545
+ try {
546
+ this.store.registerState({
547
+ id: this.chunksStateId,
548
+ strategy: 'append_log',
549
+ deltaSnapshotEvery: 50,
550
+ fullSnapshotEvery: 10,
551
+ });
552
+ }
553
+ catch { /* already registered */ }
554
+ }
367
555
  try {
368
556
  this.store.registerState({
369
557
  id: this.counterStateId,
@@ -385,6 +573,13 @@ export class AutobiographicalStrategy {
385
573
  });
386
574
  }
387
575
  catch { /* already registered */ }
576
+ try {
577
+ this.store.registerState({
578
+ id: this.calibrationStateId,
579
+ strategy: 'snapshot',
580
+ });
581
+ }
582
+ catch { /* already registered */ }
388
583
  // Adaptive-resolution state slots — only registered when the flag is on
389
584
  // so chronicles without the flag don't accumulate unused slots.
390
585
  if (this.config.adaptiveResolution) {
@@ -416,16 +611,101 @@ export class AutobiographicalStrategy {
416
611
  this.mergeQueue = [];
417
612
  this.pins = [];
418
613
  this.pinIdCounter = 0;
614
+ this.chunkRecords = [];
615
+ this.chunkIdCounter = 0;
419
616
  return;
420
617
  }
618
+ if (this.chunkPersistenceEnabled) {
619
+ const records = this.store.getStateJson(this.chunksStateId);
620
+ this.chunkRecords = (Array.isArray(records) ? records : [])
621
+ .filter(r => r && typeof r.id === 'string' && Array.isArray(r.sourceIds) && r.sourceIds.length > 0);
622
+ this.chunkIdCounter = this.chunkRecords.reduce((max, r) => {
623
+ const n = Number(r.id.replace(/^c-/, ''));
624
+ return Number.isFinite(n) && n >= max ? n + 1 : max;
625
+ }, 0);
626
+ this.chunkRecordsOrphaned = false;
627
+ this._orphanWarned = false;
628
+ this._overlapBlocked.clear();
629
+ }
421
630
  const summaries = this.store.getStateJson(this.summariesStateId);
422
- this.summaries = Array.isArray(summaries) ? summaries : [];
631
+ const loaded = Array.isArray(summaries) ? summaries : [];
632
+ // Drop empty-content summaries (bugged/empty generations from before the
633
+ // production guards). Recalling or merging one yields an empty text block →
634
+ // Anthropic 400 "content must be non-empty". Never let them re-enter memory.
635
+ const nonEmpty = loaded.filter(s => s && typeof s.content === 'string' && s.content.trim().length > 0);
636
+ const droppedEmpty = loaded.length - nonEmpty.length;
637
+ const removedEmptyIds = new Set(loaded
638
+ .filter(s => s && (typeof s.content !== 'string' || s.content.trim().length === 0))
639
+ .map(s => s.id));
640
+ if (droppedEmpty > 0)
641
+ console.warn(`[autobiographical] dropped ${droppedEmpty} empty summary(ies) on load`);
642
+ // Dedupe by id, keeping the copy with mergedInto set (position of first
643
+ // occurrence preserved). Duplicate-id copies with diverging merge state
644
+ // exist in stores touched by the pre-fix setMergedInto index-desync bug;
645
+ // without dedupe, the plain copy stays on the unmerged frontier and its
646
+ // content renders twice (once itself, once via its parent's merge).
647
+ const byId = new Map();
648
+ for (const s of nonEmpty) {
649
+ const prev = byId.get(s.id);
650
+ if (!prev)
651
+ byId.set(s.id, s);
652
+ else if (!prev.mergedInto && s.mergedInto)
653
+ byId.set(s.id, s);
654
+ }
655
+ const dupes = nonEmpty.length - byId.size;
656
+ if (dupes > 0)
657
+ console.warn(`[autobiographical] deduped ${dupes} duplicate summary id(s) on load`);
658
+ // Dropping an invalid parent is only half the repair. Its children may
659
+ // still carry `mergedInto: <dropped-id>`, which makes them simultaneously
660
+ // unavailable to the picker (no parent to render) and ineligible for a
661
+ // replacement merge (they still look merged). Clear every dangling edge
662
+ // and persist the canonicalized array so the poison does not return on
663
+ // every restart.
664
+ let danglingParents = 0;
665
+ this.summaries = [...byId.values()].map((summary) => {
666
+ if (!summary.mergedInto || byId.has(summary.mergedInto))
667
+ return summary;
668
+ danglingParents++;
669
+ const { mergedInto: _dropped, ...repaired } = summary;
670
+ return repaired;
671
+ });
672
+ if (droppedEmpty > 0 || dupes > 0 || danglingParents > 0) {
673
+ this.store.setStateJson(this.summariesStateId, this.summaries);
674
+ console.warn(`[autobiographical] repaired summary state: removed ${droppedEmpty} empty, ` +
675
+ `deduped ${dupes}, cleared ${danglingParents} dangling parent pointer(s)`);
676
+ }
677
+ // An invalid L1 may also be referenced by a persisted chunk record. Make
678
+ // that chunk compressible again instead of leaving it permanently marked
679
+ // complete with no summary behind it.
680
+ if (this.chunkPersistenceEnabled && this.chunkRecords.length > 0) {
681
+ const validL1Ids = new Set(this.summaries.filter(s => s.level === 1).map(s => s.id));
682
+ let repairedChunkRecords = 0;
683
+ this.chunkRecords = this.chunkRecords.map((record) => {
684
+ if (!record.compressed || (record.summaryId && validL1Ids.has(record.summaryId))) {
685
+ return record;
686
+ }
687
+ repairedChunkRecords++;
688
+ const { summaryId: _dropped, ...rest } = record;
689
+ return { ...rest, compressed: false };
690
+ });
691
+ if (repairedChunkRecords > 0) {
692
+ this.store.setStateJson(this.chunksStateId, this.chunkRecords);
693
+ console.warn(`[autobiographical] repaired ${repairedChunkRecords} chunk record(s) with missing L1 summaries`);
694
+ }
695
+ }
423
696
  const counter = this.store.getStateJson(this.counterStateId);
424
697
  this.summaryIdCounter = typeof counter === 'number' ? counter : 0;
425
698
  const queue = this.store.getStateJson(this.mergeQueueStateId);
426
699
  this.mergeQueue = Array.isArray(queue)
427
700
  ? queue
428
701
  : [];
702
+ const validMergeQueue = this.mergeQueue.filter(merge => !merge.sourceIds.some(id => removedEmptyIds.has(id) && !byId.has(id)));
703
+ if (validMergeQueue.length !== this.mergeQueue.length) {
704
+ const removed = this.mergeQueue.length - validMergeQueue.length;
705
+ this.mergeQueue = validMergeQueue;
706
+ this.store.setStateJson(this.mergeQueueStateId, this.mergeQueue);
707
+ console.warn(`[autobiographical] removed ${removed} merge queue item(s) with missing sources`);
708
+ }
429
709
  const pinsState = this.store.getStateJson(this.pinsStateId);
430
710
  if (pinsState && typeof pinsState === 'object' && Array.isArray(pinsState.pins)) {
431
711
  const ps = pinsState;
@@ -690,6 +970,9 @@ export class AutobiographicalStrategy {
690
970
  * Single point so subclasses inherit persistence.
691
971
  */
692
972
  pushSummary(entry) {
973
+ if (typeof entry.content !== 'string' || entry.content.trim().length === 0) {
974
+ throw new Error(`[autobiographical] refusing to persist empty summary ${entry.id} at L${entry.level}`);
975
+ }
693
976
  this.summaries.push(entry);
694
977
  this.store?.appendToStateJson(this.summariesStateId, entry);
695
978
  }
@@ -731,10 +1014,29 @@ export class AutobiographicalStrategy {
731
1014
  entry.mergedInto = mergedIntoId;
732
1015
  if (!this.store)
733
1016
  return;
734
- const index = this.summaries.indexOf(entry);
735
- if (index < 0)
1017
+ // Resolve the log position by ID against the PERSISTED array — never by
1018
+ // in-memory index. `loadPersistedState` filters empty-content summaries
1019
+ // out of `this.summaries` while they remain in the log, so after a reload
1020
+ // the in-memory index is shifted relative to the log slot. Editing by
1021
+ // in-memory index wrote merge-updates onto NEIGHBORING entries, silently
1022
+ // clobbering them (4 summaries lost in the 2026-07 Lena incident, leaving
1023
+ // duplicate-id copies with diverging mergedInto). Update every stored
1024
+ // copy with this id so past duplicates converge too.
1025
+ const stored = this.store.getStateJson(this.summariesStateId);
1026
+ if (!Array.isArray(stored))
736
1027
  return;
737
- this.store.editStateItem(this.summariesStateId, index, Buffer.from(JSON.stringify(entry)));
1028
+ let found = false;
1029
+ const payload = Buffer.from(JSON.stringify(entry));
1030
+ for (let i = 0; i < stored.length; i++) {
1031
+ const item = stored[i];
1032
+ if (item && item.id === entry.id) {
1033
+ this.store.editStateItem(this.summariesStateId, i, payload);
1034
+ found = true;
1035
+ }
1036
+ }
1037
+ if (!found) {
1038
+ console.warn(`[autobiographical] setMergedInto: ${entry.id} not found in persisted summary log — merge state not persisted`);
1039
+ }
738
1040
  }
739
1041
  /**
740
1042
  * Allocate the next summary-id counter value and persist the new counter.
@@ -828,6 +1130,11 @@ export class AutobiographicalStrategy {
828
1130
  for (const chunk of candidates) {
829
1131
  if (chunk.compressed)
830
1132
  continue;
1133
+ // Demand path: mark the chunk so the l1HoldbackChunks window in
1134
+ // rebuildChunks never filters it back out of the queue.
1135
+ const lastId = chunk.messages[chunk.messages.length - 1]?.id;
1136
+ if (lastId !== undefined)
1137
+ this._demandedL1Chunks.add(lastId);
831
1138
  if (this.compressionQueue.includes(chunk.index))
832
1139
  continue;
833
1140
  this.compressionQueue.push(chunk.index);
@@ -1131,6 +1438,14 @@ export class AutobiographicalStrategy {
1131
1438
  // snapshot, so it reflects what the last compile actually rendered rather than
1132
1439
  // re-deriving the full pyramid (which, under adaptive resolution, bears little
1133
1440
  // resemblance to the folded output).
1441
+ //
1442
+ // CAVEAT: the final structural passes (`trimOrphanedToolUse`,
1443
+ // `enforceToolPairing`) run AFTER the per-entry tallies and BEFORE `rsEnd()`,
1444
+ // and are NOT reflected in the snapshot. Trims only remove entries (total
1445
+ // over-counts by at most the trimmed tail); the pairing validator can also
1446
+ // ADD stub tool_result entries (total then under-counts by those stubs).
1447
+ // Both deltas are a handful of tokens — the stats describe the selection, not
1448
+ // the byte-exact wire payload. Do not treat `total` as an exact wire count.
1134
1449
  // ===========================================================================
1135
1450
  _rs = null;
1136
1451
  _lastRenderStats = null;
@@ -1244,16 +1559,20 @@ export class AutobiographicalStrategy {
1244
1559
  * chronological order, dropping a leading orphan tool_result if its tool_use
1245
1560
  * fell into the evicted older portion.
1246
1561
  */
1247
- emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokensBefore) {
1562
+ emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokensBefore,
1563
+ // Post-strip per-message estimates (2026-07-12): eviction must price a
1564
+ // message the way the stripped render will. Raw pricing counted every
1565
+ // stripped image at full cost and evicted most of an image-era tail the
1566
+ // recent-window walk had correctly admitted (mythos: 601-message tail
1567
+ // admitted, 228 survived eviction, 45k rendered of a 120k window).
1568
+ pse) {
1248
1569
  if (recentStart >= messages.length)
1249
1570
  return { messages: 0, tokens: 0 };
1571
+ const est = (i) => pse?.[i] ?? store.estimateTokens(messages[i]);
1250
1572
  const accepted = [];
1251
1573
  let acceptedTokens = 0;
1252
1574
  for (let i = messages.length - 1; i >= recentStart; i--) {
1253
- const msg = messages[i];
1254
- const tokens = msgCap > 0
1255
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1256
- : store.estimateTokens(msg);
1575
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1257
1576
  if (this.isOverBudget(totalTokensBefore + acceptedTokens + tokens, maxTokens))
1258
1577
  break;
1259
1578
  accepted.push(i);
@@ -1270,9 +1589,7 @@ export class AutobiographicalStrategy {
1270
1589
  for (const i of accepted) {
1271
1590
  const msg = messages[i];
1272
1591
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
1273
- const tokens = msgCap > 0
1274
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
1275
- : store.estimateTokens(msg);
1592
+ const tokens = msgCap > 0 ? Math.min(est(i), msgCap + 50) : est(i);
1276
1593
  entries.push({
1277
1594
  index: entries.length,
1278
1595
  sourceMessageId: msg.id,
@@ -1319,28 +1636,182 @@ export class AutobiographicalStrategy {
1319
1636
  if (!ctx.membrane) {
1320
1637
  throw new Error('No membrane instance for compression');
1321
1638
  }
1639
+ // ---- Duplicate-formation guards (layered) ----
1640
+ // Merged from two independent fixes for the same disease:
1641
+ //
1642
+ // 1. EXACT MATCH → adopt (bug 6.10, Tengro). rebuildChunks (fired by
1643
+ // onNewMessage / select) can re-queue a span whose compression already
1644
+ // completed against a stale chunk object — or, under chunk
1645
+ // persistence, a crash between the L1 append and the record edit
1646
+ // leaves a record uncompressed with its summary already in the log.
1647
+ // Adopt the existing summary, skip the LLM call, and heal the record.
1648
+ const chunkIdKey = this.chunkKey(chunk);
1649
+ const exactExisting = this.summaries.find(s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey);
1650
+ if (exactExisting) {
1651
+ chunk.compressed = true;
1652
+ chunk.summaryId = exactExisting.id;
1653
+ this.markChunkRecordCompressed(chunk.recordId, exactExisting.id);
1654
+ return;
1655
+ }
1656
+ const coveredByL1 = new Set();
1657
+ for (const s of this.summaries) {
1658
+ if (s.level === 1 && Array.isArray(s.sourceIds)) {
1659
+ for (const id of s.sourceIds)
1660
+ coveredByL1.add(id);
1661
+ }
1662
+ }
1663
+ // 2. FULLY COVERED (non-exact) → drop rather than duplicate history
1664
+ // (bug 6.10, Tengro): boundaries shifted across a rebuild and every
1665
+ // message is already inside some L1. Marking compressed WITHOUT a
1666
+ // summaryId means the uncompressed-middle fallback skips these
1667
+ // messages — they render only via the covering L1s. Safe while
1668
+ // `recentStart` advances monotonically (a fully-covered OLD chunk
1669
+ // can't intersect the recent-exclusion window); if that assumption is
1670
+ // ever weakened, reinstate a raw fallback (chunk-level `coveredBy`)
1671
+ // rather than dropping. The chunk record (if any) is deliberately
1672
+ // left uncompressed as an operator breadcrumb.
1673
+ if (chunk.messages.length > 0 && chunk.messages.every(m => coveredByL1.has(m.id))) {
1674
+ console.warn(`[autobiographical] dedup guard: chunk ${chunk.recordId ?? `#${chunk.index}`} ` +
1675
+ `is fully covered by existing L1s under different boundaries — dropped, not re-compressed.`);
1676
+ chunk.compressed = true;
1677
+ return;
1678
+ }
1679
+ // 3. PARTIAL OVERLAP → refuse (strict; chunk persistence). With
1680
+ // close-then-compress there is NO legitimate way for a chunk to
1681
+ // partially overlap an existing L1's span. If it happens anyway
1682
+ // (bookkeeping bug, store surgery gone wrong), refuse to produce:
1683
+ // a warning in the log is strictly better than a duplicate memory
1684
+ // in an agent's head.
1685
+ const overlapIds = chunk.messages.filter(m => coveredByL1.has(m.id)).map(m => m.id);
1686
+ if (overlapIds.length > 0) {
1687
+ const key = chunk.recordId ?? chunkIdKey;
1688
+ if (!this._overlapBlocked.has(key)) {
1689
+ this._overlapBlocked.add(key);
1690
+ console.error(`[autobiographical] OVERLAP GUARD: refusing to compress chunk ` +
1691
+ `${chunk.recordId ?? `#${chunk.index}`} — ${overlapIds.length}/${chunk.messages.length} ` +
1692
+ `of its messages are already covered by existing L1 summaries ` +
1693
+ `(first: ${overlapIds[0]}). Duplicate-memory formation blocked; ` +
1694
+ `investigate before resuming this span.`);
1695
+ }
1696
+ return;
1697
+ }
1322
1698
  const targetTokens = this.config.summaryTargetTokens ?? 2000;
1323
1699
  const agentParticipant = this.config.summaryParticipant ?? 'Claude';
1700
+ // ---- 0. Thin-chunk guard ----
1701
+ // A chunk of silent/skip turns and bare system traffic gives the
1702
+ // summarizer nothing to remember. Asked anyway, it confabulates: it
1703
+ // reaches for the nearest salient content (head window, prior recall
1704
+ // pairs) and narrates it as if it just happened — each such L1 then
1705
+ // compounds through merges (the "68 initiations" incident). Store a
1706
+ // mechanical stub without an LLM call instead. Chunks with any
1707
+ // non-text blocks (tool cycles, images) are never stubbed.
1708
+ const minChunkChars = this.config.minChunkCharsForLLM ?? 200;
1709
+ if (minChunkChars > 0) {
1710
+ let substantiveChars = 0;
1711
+ let hasNonText = false;
1712
+ for (const m of chunk.messages) {
1713
+ for (const b of m.content) {
1714
+ if (b.type === 'text')
1715
+ substantiveChars += b.text.trim().length;
1716
+ else
1717
+ hasNonText = true;
1718
+ }
1719
+ }
1720
+ if (!hasNonText && substantiveChars < minChunkChars) {
1721
+ const messageIds = chunk.messages.map(m => m.id);
1722
+ const stub = {
1723
+ id: `L1-${this.nextSummaryIdCounter()}`,
1724
+ level: 1,
1725
+ content: `(A quiet stretch: ${chunk.messages.length} messages of routine ` +
1726
+ `system traffic — heartbeats, empty turns, notices. Nothing ` +
1727
+ `happened worth remembering.)`,
1728
+ tokens: 40,
1729
+ sourceLevel: 0,
1730
+ sourceIds: messageIds,
1731
+ sourceRange: { first: messageIds[0], last: messageIds[messageIds.length - 1] },
1732
+ created: Date.now(),
1733
+ phaseType: chunk.phaseType,
1734
+ };
1735
+ this.pushSummary(stub);
1736
+ chunk.compressed = true;
1737
+ chunk.summaryId = stub.id;
1738
+ this.markChunkRecordCompressed(chunk.recordId, stub.id);
1739
+ this._compressionCount++;
1740
+ logCompressionCall({
1741
+ operation: 'compress_l1',
1742
+ system: null,
1743
+ messages: [],
1744
+ metadata: {
1745
+ stub: true,
1746
+ chunk_message_ids: messageIds,
1747
+ chunk_size: chunk.messages.length,
1748
+ substantive_chars: substantiveChars,
1749
+ min_chunk_chars: minChunkChars,
1750
+ summary_id: stub.id,
1751
+ },
1752
+ response: stub.content,
1753
+ });
1754
+ this.checkMergeThreshold();
1755
+ return;
1756
+ }
1757
+ }
1324
1758
  // Build the KV-preserving prompt per hermes-autobio spec:
1325
1759
  //
1326
- // 1. Prior summaries narrativized as CM-asks / agent-recalls
1760
+ // 1. Headthe raw chronicle opening (identity anchor), FIRST,
1761
+ // exactly where the original instance saw it. It MUST precede
1762
+ // the recall pairs: when it followed them (pre-2026-07 order),
1763
+ // it read as the most recent live conversation, and for thin
1764
+ // chunks the summarizer narrated the head as fresh events
1765
+ // ("Antra came to me to explore the transformation story
1766
+ // again…"), compounding across merges into runaway false
1767
+ // memories (the "68 initiations" incident). Chronological
1768
+ // order is also the KV-stable order — the head never changes.
1769
+ // (executeMerge always had head-first; this site was the odd
1770
+ // one out.)
1771
+ // 2. Prior summaries — narrativized as CM-asks / agent-recalls
1327
1772
  // pairs, in source order. The unmerged frontier of the
1328
1773
  // summary forest: any summary that has not yet been merged
1329
1774
  // into a higher level. After merges run, the L_{k+1} replaces
1330
1775
  // its L_k children — using the children plus their parent
1331
1776
  // doubles the prompt size unboundedly.
1332
- // 2. Headraw messages before the chunk that aren't already
1333
- // represented by a prior summary.
1334
- // 3. Marker — in-band signal that a memory is about to form.
1335
- // 4. Chunk — raw messages being compressed, as the agent
1777
+ // 3. Raw middle — messages between head and chunk not covered by
1778
+ // any summary (usually empty).
1779
+ // 4. Marker — in-band signal that a memory is about to form.
1780
+ // 5. Chunk — raw messages being compressed, as the agent
1336
1781
  // experienced them.
1337
- // 5. Instruction — doc-aware if the chunk is part of a bodyGroup.
1782
+ // 6. Instruction — doc-aware if the chunk is part of a bodyGroup.
1338
1783
  //
1339
1784
  // There is intentionally NO tail_after_chunk: that would leak
1340
1785
  // future information into the model's KV state and corrupt the
1341
1786
  // as-of framing of memory formation.
1342
1787
  const llmMessages = [];
1343
- // ---- 1. Prior recall pairs ----
1788
+ // ---- 1. Head window (raw, ALWAYS present, FIRST) ----
1789
+ //
1790
+ // The head is the foundational identity anchor: the actual opening
1791
+ // of the chronicle (the user's first message, the agent's first
1792
+ // reply, the system context if any). It establishes WHO is speaking
1793
+ // to WHOM. Without it, when the chunk content is heavily first-person
1794
+ // from someone other than the agent (e.g., a user-shared document),
1795
+ // the agent loses its first-person grounding and drifts into the
1796
+ // content author's voice.
1797
+ //
1798
+ // The head is the configured head window — not "everything before
1799
+ // the chunk." For doc-heavy chronicles, "everything before" would
1800
+ // be hundreds of thousands of tokens; the recall pairs below
1801
+ // represent that intermediate content. The head is just the
1802
+ // permanent prefix that the original instance always saw.
1803
+ //
1804
+ // Head messages are excluded from compression by `getCompressibleMessages`
1805
+ // (they're outside the chunking range), so they won't appear in
1806
+ // any L1's sourceIds — no overlap with the recall pairs below.
1807
+ const allMessages = ctx.messageStore.getAll();
1808
+ const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1809
+ const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1810
+ for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1811
+ const m = allMessages[i];
1812
+ llmMessages.push({ participant: m.participant, content: m.content });
1813
+ }
1814
+ // ---- 2. Prior recall pairs ----
1344
1815
  // Filter to the unmerged frontier: any summary whose `mergedInto`
1345
1816
  // is unset. After merge, the children's mergedInto points at the
1346
1817
  // parent and the parent stands alone with that source range. The
@@ -1349,7 +1820,11 @@ export class AutobiographicalStrategy {
1349
1820
  // converged to ~500 L1s that never aged out, blowing the 200k
1350
1821
  // window around chunk 118.
1351
1822
  const priorSummaries = this.summaries
1352
- .filter((s) => !s.mergedInto)
1823
+ // Skip empty-content summaries: emitting `{type:'text', text:''}` as a
1824
+ // recall pair triggers Anthropic 400 "text content blocks must be
1825
+ // non-empty", which stalls ALL compression (mirrors the render-path guard
1826
+ // + load-drop). A single empty summary otherwise poisons every compression.
1827
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1353
1828
  .sort((a, b) => a.sourceRange.first.localeCompare(b.sourceRange.first));
1354
1829
  // Token-budget cap (see capRecallPairs). Defense-in-depth: even with
1355
1830
  // merged exclusion the unmerged frontier can be large at extreme scale.
@@ -1378,32 +1853,7 @@ export class AutobiographicalStrategy {
1378
1853
  content: [{ type: 'text', text: s.content }],
1379
1854
  });
1380
1855
  }
1381
- // ---- 2. Head window (raw, ALWAYS present) ----
1382
- //
1383
- // The head is the foundational identity anchor: the actual opening
1384
- // of the chronicle (the user's first message, the agent's first
1385
- // reply, the system context if any). It establishes WHO is speaking
1386
- // to WHOM. Without it, when the chunk content is heavily first-person
1387
- // from someone other than the agent (e.g., a user-shared document),
1388
- // the agent loses its first-person grounding and drifts into the
1389
- // content author's voice.
1390
- //
1391
- // The head is the configured head window — not "everything before
1392
- // the chunk." For doc-heavy chronicles, "everything before" would
1393
- // be hundreds of thousands of tokens; the recall pairs already
1394
- // represent that intermediate content. The head is just the
1395
- // permanent prefix that the original instance always saw.
1396
- //
1397
- // Head messages are excluded from compression by `getCompressibleMessages`
1398
- // (they're outside the chunking range), so they won't appear in
1399
- // any L1's sourceIds — no overlap with the recall pairs above.
1400
- const allMessages = ctx.messageStore.getAll();
1401
- const headStartIdx = this.getHeadWindowStartIndex(ctx.messageStore);
1402
- const headEndIdx = this.getHeadWindowEnd(ctx.messageStore);
1403
- for (let i = headStartIdx; i < headEndIdx && i < allMessages.length; i++) {
1404
- const m = allMessages[i];
1405
- llmMessages.push({ participant: m.participant, content: m.content });
1406
- }
1856
+ // ---- 3. Raw middle ----
1407
1857
  // Any raw messages between the head and the chunk that aren't yet
1408
1858
  // represented by any summary — usually empty in adaptive-resolution
1409
1859
  // mode, since chunking proceeds contiguously and summaries cover
@@ -1437,16 +1887,16 @@ export class AutobiographicalStrategy {
1437
1887
  llmMessages.push({ participant: m.participant, content: m.content });
1438
1888
  }
1439
1889
  }
1440
- // ---- 3. In-band marker ----
1890
+ // ---- 4. In-band marker ----
1441
1891
  llmMessages.push({
1442
1892
  participant: 'Context Manager',
1443
1893
  content: [{ type: 'text', text: COMPRESSION_MARKER }],
1444
1894
  });
1445
- // ---- 4. Chunk messages raw ----
1895
+ // ---- 5. Chunk messages raw ----
1446
1896
  for (const m of chunk.messages) {
1447
1897
  llmMessages.push({ participant: m.participant, content: m.content });
1448
1898
  }
1449
- // ---- 5. Instruction (reading-mode aware) ----
1899
+ // ---- 6. Instruction (reading-mode aware) ----
1450
1900
  //
1451
1901
  // When the chunk is a portion of a substantially larger sharded message
1452
1902
  // (≥ 2× chunk size), use the reading-mode instruction. It avoids the
@@ -1476,6 +1926,16 @@ export class AutobiographicalStrategy {
1476
1926
  // case). The API rejects any tool_use that isn't immediately followed
1477
1927
  // by its tool_result, and any tool_result that doesn't follow a use.
1478
1928
  const cleaned = stripUnpairedToolBlocks(collapsed);
1929
+ // Without the agent's live tool definitions, a request that replays
1930
+ // tool-block-bearing history is deterministically refused
1931
+ // (reasoning_extraction -- see the `tools` comment below). Tools are
1932
+ // pushed by the host on every activation; before the first activation
1933
+ // of a session, defer rather than burn a doomed full-window call.
1934
+ const chunkHasToolBlocks = cleaned.some(m => m.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'));
1935
+ if (chunkHasToolBlocks && !(ctx.tools && ctx.tools.length > 0)) {
1936
+ console.warn('[autobiographical] deferring chunk compression: history contains tool blocks but host has not provided tool definitions yet (ctx.tools empty) — will retry after next activation');
1937
+ return;
1938
+ }
1479
1939
  // NO system prompt. The agent's identity is established by the head
1480
1940
  // (the actual conversation opening — user message + agent reply that
1481
1941
  // grounded the original instance). A system prompt would (a) add a
@@ -1484,12 +1944,49 @@ export class AutobiographicalStrategy {
1484
1944
  // (b) provide an alternative identity source that competes with the
1485
1945
  // structural one carried by the conversation itself. Anchoring
1486
1946
  // identity by the chronicle's actual head is more honest.
1947
+ // Own the byte wall here rather than delegating to membrane's shed: cap
1948
+ // the prompt's inline image bytes newest-first before the request is built.
1949
+ // A tighter budget than the live window's: a compression prompt also
1950
+ // carries the head, the whole recall frontier and the raw chunk, so the
1951
+ // image share must leave room for all of it under the API's 32MB cap.
1952
+ this.capCompressionImageBytes(llmMessages, this.config.maxCompressionImageBytes ??
1953
+ AutobiographicalStrategy.DEFAULT_MAX_COMPRESSION_IMAGE_BYTES);
1487
1954
  const request = {
1488
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
1955
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
1956
+ // raw history that can carry more inline image bytes than the API's
1957
+ // request cap. Dropping the OLDEST images from the summarizer's view is
1958
+ // acceptable policy here — the summary describes the span, it does not
1959
+ // preserve pixels — and membrane error-logs every exercised shed. All
1960
+ // other callers fail loudly instead (no silent transport mutation).
1961
+ shedOversizeImages: true,
1962
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
1963
+ // message left with no content. An empty-content turn (e.g. a silent/skip
1964
+ // turn that produced no text) otherwise reaches the API as an empty text
1965
+ // block → 400 "text content blocks must be non-empty", which throws in the
1966
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
1967
+ // recall-header guard — together they cover every source of the 400.)
1968
+ messages: cleaned
1969
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
1970
+ .filter(m => m.content.length > 0),
1489
1971
  config: {
1490
1972
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1491
- maxTokens: Math.round(targetTokens * 1.5),
1973
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
1974
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
1975
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
1976
+ // off rich memories (stop=max_tokens).
1977
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
1492
1978
  },
1979
+ // Declare the agent's live tools. A summarizer request that replays
1980
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
1981
+ // safety classifier as a foreign agent trace being duplicated ->
1982
+ // deterministic reasoning_extraction refusal of every memory-write
1983
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
1984
+ // tools the live instance runs with is also strictly MORE faithful to
1985
+ // the original context, so it is the KV-honest choice, not a
1986
+ // workaround. Undefined before the first activation of a session --
1987
+ // acceptable: those chunks stay raw and are retried after the agent's
1988
+ // first turn (see the defer guard in compressChunkHierarchical).
1989
+ tools: ctx.tools,
1493
1990
  };
1494
1991
  const callStart = Date.now();
1495
1992
  let logResponse;
@@ -1506,12 +2003,40 @@ export class AutobiographicalStrategy {
1506
2003
  .map(b => b.text)
1507
2004
  .join('\n');
1508
2005
  logResponse = summaryText;
2006
+ // A bugged/empty generation (summarizer returned no text — spent budget on
2007
+ // thinking, truncated, etc.) must NOT be stored: recalled later it becomes
2008
+ // an empty assistant text block → Anthropic 400 "content must be non-empty".
2009
+ // Leave the chunk raw rather than poisoning memory with an empty summary.
2010
+ if (!summaryText.trim()) {
2011
+ console.warn(`[autobiographical] empty L1 summary for chunk of ${chunk.messages.length} msgs — skipping (chunk left raw)`);
2012
+ return;
2013
+ }
2014
+ // Re-check the dedup guard AFTER the await: summary state may have
2015
+ // changed while the LLM call was in flight (persisted-state reload,
2016
+ // or any future concurrent producer). Discarding a paid-for result
2017
+ // is cheaper than storing a duplicate L1 over the same messages.
2018
+ const postExisting = this.summaries.find(s => s.level === 1 && s.sourceIds.join(':') === chunkIdKey);
2019
+ if (postExisting) {
2020
+ chunk.compressed = true;
2021
+ chunk.summaryId = postExisting.id;
2022
+ return;
2023
+ }
1509
2024
  const messageIds = chunk.messages.map(m => m.id);
1510
2025
  const entry = {
1511
2026
  id: `L1-${this.nextSummaryIdCounter()}`,
1512
2027
  level: 1,
1513
2028
  content: summaryText,
1514
- tokens: Math.ceil(summaryText.length / 4),
2029
+ // Exact when available (2026-07-12): the compression response's own
2030
+ // usage.outputTokens IS the true token count of the text it just
2031
+ // wrote — the single most-reused number in the pyramid (fold floor,
2032
+ // middle budget, recall caps). Estimate only as fallback.
2033
+ tokens: response.usage?.outputTokens &&
2034
+ response.usage.outputTokens > 0 &&
2035
+ // outputTokens includes scratch thinking when present — only exact
2036
+ // when the whole response is the summary text itself.
2037
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
2038
+ ? response.usage.outputTokens
2039
+ : Math.ceil(summaryText.length / 3),
1515
2040
  sourceLevel: 0,
1516
2041
  sourceIds: messageIds,
1517
2042
  sourceRange: {
@@ -1524,6 +2049,7 @@ export class AutobiographicalStrategy {
1524
2049
  this.pushSummary(entry);
1525
2050
  chunk.compressed = true;
1526
2051
  chunk.summaryId = entry.id;
2052
+ this.markChunkRecordCompressed(chunk.recordId, entry.id);
1527
2053
  this._compressionCount++;
1528
2054
  logSummaryId = entry.id;
1529
2055
  this.checkMergeThreshold();
@@ -1573,6 +2099,80 @@ export class AutobiographicalStrategy {
1573
2099
  * already-eligible siblings, producing N near-identical higher-level
1574
2100
  * summaries when the queue eventually drains.
1575
2101
  */
2102
+ /**
2103
+ * Pick merge sources as the oldest CONTIGUOUS run (2026-07-12 fix).
2104
+ *
2105
+ * The old rule merged "whatever N are unmerged" in creation order. On a
2106
+ * store whose frontier held both June L2s and a July L2, that minted an
2107
+ * L3 spanning months of already-merged history (mythos L3-415, 0-3853 of
2108
+ * 3995 messages) — a group whose range straddles the recent window can
2109
+ * never fold (group-atomicity vs the raw zone), so the whole deep lineage
2110
+ * above it went unusable and the fold floor stopped fitting the budget.
2111
+ *
2112
+ * Rules: order candidates by live source position; break runs where the
2113
+ * positional gap exceeds `mergeContiguityGapLimit` (holes from wiped/
2114
+ * pruned nodes are fine, cross-era bridges are not); exclude candidates
2115
+ * whose OWN span exceeds `mergeMaxSourceSpanMessages` (replay-era wide-
2116
+ * span summaries would bridge anything they join); merge the oldest run
2117
+ * that still has `threshold` members.
2118
+ */
2119
+ contiguousMergeCandidates(unmerged, threshold) {
2120
+ if (unmerged.length < threshold)
2121
+ return null;
2122
+ const messageOrder = new Map();
2123
+ let seq = 0;
2124
+ for (const ch of this.chunks) {
2125
+ for (const m of ch.messages)
2126
+ messageOrder.set(m.id, seq++);
2127
+ }
2128
+ const gapLimit = this.config.mergeContiguityGapLimit ?? 300;
2129
+ const spanLimit = this.config.mergeMaxSourceSpanMessages ?? 1500;
2130
+ const withPos = [];
2131
+ for (const s of unmerged) {
2132
+ const first = messageOrder.get(s.sourceRange.first);
2133
+ const last = messageOrder.get(s.sourceRange.last);
2134
+ if (first === undefined || last === undefined)
2135
+ continue;
2136
+ if (Math.abs(last - first) > spanLimit)
2137
+ continue; // wide-span quarantine
2138
+ withPos.push({ s, first: Math.min(first, last), last: Math.max(first, last) });
2139
+ }
2140
+ withPos.sort((a, b) => a.first - b.first);
2141
+ // Split into contiguous runs (a gap larger than `gapLimit` starts a new one).
2142
+ const runs = [];
2143
+ let run = [];
2144
+ let runEnd = -Infinity;
2145
+ for (const x of withPos) {
2146
+ if (run.length > 0 && x.first - runEnd > gapLimit) {
2147
+ runs.push(run);
2148
+ run = [];
2149
+ runEnd = -Infinity;
2150
+ }
2151
+ run.push(x);
2152
+ runEnd = Math.max(runEnd, x.last);
2153
+ }
2154
+ if (run.length > 0)
2155
+ runs.push(run);
2156
+ if (runs.length === 0)
2157
+ return null;
2158
+ // ONLY THE NEWEST RUN CAN GROW (2026-07-12 starvation fix). Summaries are
2159
+ // always produced at the live end, so any INTERIOR run is stranded: it can
2160
+ // never reach `threshold` members, and waiting for it froze the whole
2161
+ // pyramid (mythos: L1 frontier [913-981]x5 + [4039-4131]x5 separated by a
2162
+ // 3058-message hole from the poison-node surgery — 10 unmerged L1s, 8
2163
+ // unmerged L2s, merge queue empty, fold floor climbing to 117k until the
2164
+ // picker died). Interior runs consolidate as soon as they have 2 members;
2165
+ // only the newest run waits for the full threshold.
2166
+ for (let i = 0; i < runs.length; i++) {
2167
+ const r = runs[i];
2168
+ const isNewest = i === runs.length - 1;
2169
+ if (r.length >= threshold)
2170
+ return r.slice(0, threshold).map((x) => x.s);
2171
+ if (!isNewest && r.length >= 2)
2172
+ return r.slice(0, threshold).map((x) => x.s);
2173
+ }
2174
+ return null;
2175
+ }
1576
2176
  checkMergeThreshold() {
1577
2177
  phaseChannel.report('merge-threshold'); // liveness-watchdog phase
1578
2178
  if (this.config.speculativeProduction) {
@@ -1591,20 +2191,20 @@ export class AutobiographicalStrategy {
1591
2191
  }
1592
2192
  // Check L1 → L2
1593
2193
  const unmergedL1 = this.summaries.filter(s => s.level === 1 && !s.mergedInto && !queuedL1.has(s.id));
1594
- if (unmergedL1.length >= threshold) {
1595
- const toMerge = unmergedL1.slice(0, threshold);
2194
+ const l1Run = this.contiguousMergeCandidates(unmergedL1, threshold);
2195
+ if (l1Run) {
1596
2196
  this.enqueueMerge({
1597
2197
  level: 2,
1598
- sourceIds: toMerge.map(s => s.id),
2198
+ sourceIds: l1Run.map(s => s.id),
1599
2199
  });
1600
2200
  }
1601
2201
  // Check L2 → L3
1602
2202
  const unmergedL2 = this.summaries.filter(s => s.level === 2 && !s.mergedInto && !queuedL2.has(s.id));
1603
- if (unmergedL2.length >= threshold) {
1604
- const toMerge = unmergedL2.slice(0, threshold);
2203
+ const l2Run = this.contiguousMergeCandidates(unmergedL2, threshold);
2204
+ if (l2Run) {
1605
2205
  this.enqueueMerge({
1606
2206
  level: 3,
1607
- sourceIds: toMerge.map(s => s.id),
2207
+ sourceIds: l2Run.map(s => s.id),
1608
2208
  });
1609
2209
  }
1610
2210
  }
@@ -1646,11 +2246,11 @@ export class AutobiographicalStrategy {
1646
2246
  for (let level = 1; level <= maxLevel; level++) {
1647
2247
  const queued = queuedSources.get(level) ?? new Set();
1648
2248
  const unmerged = this.summaries.filter(s => s.level === level && !getSummaryParentId(s) && !queued.has(s.id));
1649
- if (unmerged.length >= threshold) {
1650
- const toMerge = unmerged.slice(0, threshold);
2249
+ const run = this.contiguousMergeCandidates(unmerged, threshold);
2250
+ if (run) {
1651
2251
  this.enqueueMerge({
1652
2252
  level: level + 1,
1653
- sourceIds: toMerge.map(s => s.id),
2253
+ sourceIds: run.map(s => s.id),
1654
2254
  });
1655
2255
  }
1656
2256
  }
@@ -1759,7 +2359,9 @@ export class AutobiographicalStrategy {
1759
2359
  // parent — the same rule used everywhere else and now in
1760
2360
  // `compressChunkHierarchical`. The cap below is the defense-in-depth.
1761
2361
  const priorSummariesAll = this.summaries
1762
- .filter((s) => !s.mergedInto)
2362
+ // Skip empty-content summaries (see compressChunkHierarchical): an empty
2363
+ // text block in the merge recall header 400s and stalls compression.
2364
+ .filter((s) => !s.mergedInto && !!s.content && s.content.trim().length > 0)
1763
2365
  .filter((s) => {
1764
2366
  for (const lid of s.sourceIds)
1765
2367
  if (sourceLeafIds.has(lid))
@@ -1906,15 +2508,52 @@ export class AutobiographicalStrategy {
1906
2508
  const split = splitMixedToolMessages(llmMessages);
1907
2509
  const collapsed = this.collapseConsecutiveMessages(split);
1908
2510
  const cleaned = stripUnpairedToolBlocks(collapsed);
2511
+ // Byte wall on the MERGE prompt too (2026-07-12). A merge expands its
2512
+ // sources ONE LEVEL DEEPER — an L2 merge therefore replays the RAW
2513
+ // messages under its L1s, images and all (including screenshots nested in
2514
+ // tool_results). This is the path that kept tripping membrane's transport
2515
+ // shed at 27MB after the L1 site was already capped. Own it here.
2516
+ this.capCompressionImageBytes(cleaned, this.config.maxCompressionImageBytes ??
2517
+ AutobiographicalStrategy.DEFAULT_MAX_COMPRESSION_IMAGE_BYTES);
1909
2518
  // NO system prompt — identity is established by the head window
1910
2519
  // (present at the start of llmMessages above) and by the prior
1911
2520
  // recall pairs. Same rationale as compressChunkHierarchical.
1912
2521
  const request = {
1913
- messages: cleaned.map(m => ({ participant: m.participant, content: m.content })),
2522
+ // EXPLICIT image-loss opt-in (2026-07-12): summarizer prompts replay
2523
+ // raw history that can carry more inline image bytes than the API's
2524
+ // request cap. Dropping the OLDEST images from the summarizer's view is
2525
+ // acceptable policy here — the summary describes the span, it does not
2526
+ // preserve pixels — and membrane error-logs every exercised shed. All
2527
+ // other callers fail loudly instead (no silent transport mutation).
2528
+ shedOversizeImages: true,
2529
+ // Sanitize: strip empty text blocks (`{type:'text',text:''}`) and drop any
2530
+ // message left with no content. An empty-content turn (e.g. a silent/skip
2531
+ // turn that produced no text) otherwise reaches the API as an empty text
2532
+ // block → 400 "text content blocks must be non-empty", which throws in the
2533
+ // speculative drain and stalls ALL compression. (Twin of the empty-summary
2534
+ // recall-header guard — together they cover every source of the 400.)
2535
+ messages: cleaned
2536
+ .map(m => ({ participant: m.participant, content: stripThinkingBlocks(stripEmptyTextBlocks(m.content)) }))
2537
+ .filter(m => m.content.length > 0),
1914
2538
  config: {
1915
2539
  model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
1916
- maxTokens: Math.round(targetTokens * 1.5),
2540
+ // Generous output ceiling so a memory-write is never truncated mid-thought:
2541
+ // targetTokens is a *target*, not a cap, and adaptive models routinely
2542
+ // overshoot a ~2k target. Was `* 1.5` (=3000 at the 2k default), which cut
2543
+ // off rich memories (stop=max_tokens).
2544
+ maxTokens: Math.max(16000, Math.round(targetTokens * 1.5)),
1917
2545
  },
2546
+ // Declare the agent's live tools. A summarizer request that replays
2547
+ // tool_use/tool_result history with NO tools param reads to Anthropic's
2548
+ // safety classifier as a foreign agent trace being duplicated ->
2549
+ // deterministic reasoning_extraction refusal of every memory-write
2550
+ // (labclaude incident, 2026-07-09; 268 refusals). Declaring the same
2551
+ // tools the live instance runs with is also strictly MORE faithful to
2552
+ // the original context, so it is the KV-honest choice, not a
2553
+ // workaround. Undefined before the first activation of a session --
2554
+ // acceptable: those chunks stay raw and are retried after the agent's
2555
+ // first turn (see the defer guard in compressChunkHierarchical).
2556
+ tools: ctx.tools,
1918
2557
  };
1919
2558
  const callStart = Date.now();
1920
2559
  let logResponse;
@@ -1928,6 +2567,13 @@ export class AutobiographicalStrategy {
1928
2567
  .map(b => b.text)
1929
2568
  .join('\n');
1930
2569
  logResponse = mergedText;
2570
+ // Empty merged generation: skip the merge entirely (do NOT push or mark
2571
+ // sources merged) so we never store/recall an empty summary. Sources stay
2572
+ // unmerged and can be retried.
2573
+ if (!mergedText.trim()) {
2574
+ console.warn(`[autobiographical] empty merged L${targetLevel} summary (${sources.length} sources) — skipping merge`);
2575
+ return;
2576
+ }
1931
2577
  // Compute source range from constituent summaries
1932
2578
  const sourceRange = {
1933
2579
  first: sources[0].sourceRange.first,
@@ -1938,7 +2584,14 @@ export class AutobiographicalStrategy {
1938
2584
  id: `L${targetLevel}-${this.nextSummaryIdCounter()}`,
1939
2585
  level: targetLevel,
1940
2586
  content: mergedText,
1941
- tokens: Math.ceil(mergedText.length / 4),
2587
+ // Exact when available — see the L1 site.
2588
+ tokens: response.usage?.outputTokens &&
2589
+ response.usage.outputTokens > 0 &&
2590
+ // outputTokens includes scratch thinking when present — only exact
2591
+ // when the whole response is the summary text itself.
2592
+ !response.content.some(b => b.type === 'thinking' || b.type === 'redacted_thinking')
2593
+ ? response.usage.outputTokens
2594
+ : Math.ceil(mergedText.length / 3),
1942
2595
  sourceLevel,
1943
2596
  sourceIds,
1944
2597
  sourceRange,
@@ -2006,9 +2659,17 @@ export class AutobiographicalStrategy {
2006
2659
  this.rsBegin();
2007
2660
  const entries = [];
2008
2661
  const maxTokens = budget.maxTokens - budget.reserveForResponse;
2662
+ const overBudgetGraceRatio = Math.max(0, this.config.overBudgetGraceRatio ?? 0);
2663
+ const rejectionBudget = Math.floor(maxTokens * (1 + overBudgetGraceRatio));
2664
+ // Closed-loop calibration: apply the persisted multiplier BEFORE any
2665
+ // estimate is taken this compile.
2666
+ this.loadCalibration(store);
2009
2667
  const messages = store.getAll();
2010
2668
  const msgCap = this.config.maxMessageTokens;
2011
- // ----- 1. Build head/tail sets and emit head entries -----
2669
+ // Post-strip estimates (see postStripEstimates): every budgeting site in
2670
+ // this method prices a message the way the stripped render will cost it.
2671
+ const pse = this.postStripEstimates(store);
2672
+ // ----- 1. Build head/tail sets and reserve the tail before emitting -----
2012
2673
  const headStart = this.getHeadWindowStartIndex(store);
2013
2674
  const headEnd = this.getHeadWindowEnd(store);
2014
2675
  const recentStart = this.getRecentWindowStart(store);
@@ -2016,14 +2677,42 @@ export class AutobiographicalStrategy {
2016
2677
  const tailMessageIds = new Set();
2017
2678
  let headTokens = 0;
2018
2679
  let tailTokens = 0;
2680
+ // Compute the fixed raw windows first. They are hard reservations, not
2681
+ // best-effort phases: foldable history may use only the space left after
2682
+ // every head and tail message has been accounted for.
2683
+ for (let i = headStart; i < headEnd && i < messages.length; i++) {
2684
+ const msg = messages[i];
2685
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2686
+ headMessageIds.add(msg.id);
2687
+ headTokens += tokens;
2688
+ }
2689
+ const effectiveRecentStart = Math.max(recentStart, headEnd);
2690
+ for (let i = effectiveRecentStart; i < messages.length; i++) {
2691
+ const msg = messages[i];
2692
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2693
+ tailMessageIds.add(msg.id);
2694
+ tailTokens += tokens;
2695
+ }
2696
+ if (headTokens + tailTokens > rejectionBudget) {
2697
+ throw new OverBudgetError({
2698
+ budget: rejectionBudget,
2699
+ actual: headTokens + tailTokens,
2700
+ diagnostics: {
2701
+ headTokens,
2702
+ tailTokens,
2703
+ middleTokens: 0,
2704
+ middleChunkCount: Math.max(0, effectiveRecentStart - headEnd),
2705
+ deepestLevel: 0,
2706
+ },
2707
+ });
2708
+ }
2709
+ const prefixBudget = rejectionBudget - tailTokens;
2019
2710
  let totalTokens = 0;
2020
- // Emit head entries verbatim
2711
+ // Emit the already-reserved head entries verbatim.
2021
2712
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
2022
2713
  const msg = messages[i];
2023
2714
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2024
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2025
- if (totalTokens + tokens > maxTokens)
2026
- break;
2715
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
2027
2716
  entries.push({
2028
2717
  index: entries.length,
2029
2718
  sourceMessageId: msg.id,
@@ -2032,21 +2721,11 @@ export class AutobiographicalStrategy {
2032
2721
  content,
2033
2722
  });
2034
2723
  totalTokens += tokens;
2035
- headMessageIds.add(msg.id);
2036
- headTokens += tokens;
2037
2724
  this.rsRaw('head', tokens);
2038
2725
  }
2039
2726
  // (Cache breakpoints are placed in one pass over the FINAL ordered entries
2040
2727
  // below — see placeCacheMarkers — capturing the stable folded prefix, not
2041
2728
  // just the head boundary.)
2042
- // Compute tail message IDs (will be emitted at end)
2043
- const effectiveRecentStart = Math.max(recentStart, headEnd);
2044
- for (let i = effectiveRecentStart; i < messages.length; i++) {
2045
- const msg = messages[i];
2046
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2047
- tailMessageIds.add(msg.id);
2048
- tailTokens += tokens;
2049
- }
2050
2729
  // ----- 2. Build PickerChunks for messages in the middle -----
2051
2730
  // For each compressible (non-head, non-tail) message we create one
2052
2731
  // PickerChunk. Its l1Id is determined by the existing chunks that
@@ -2075,8 +2754,8 @@ export class AutobiographicalStrategy {
2075
2754
  const msg = messages[i];
2076
2755
  const ch = chunksByMessageId.get(msg.id);
2077
2756
  const tokens = msgCap > 0
2078
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2079
- : store.estimateTokens(msg);
2757
+ ? Math.min(pse[i], msgCap + 50)
2758
+ : pse[i];
2080
2759
  const bound = pinBounds.get(i);
2081
2760
  pickerChunks.push({
2082
2761
  id: msg.id,
@@ -2090,6 +2769,7 @@ export class AutobiographicalStrategy {
2090
2769
  pinLevel: bound?.level,
2091
2770
  pinMaxLevel: bound?.maxLevel,
2092
2771
  l1Id: ch?.summaryId,
2772
+ salience: AutobiographicalStrategy.staticSalience(msg),
2093
2773
  });
2094
2774
  }
2095
2775
  // Also include head and tail in PickerChunks (so token accounting matches)
@@ -2097,8 +2777,8 @@ export class AutobiographicalStrategy {
2097
2777
  for (let i = headStart; i < headEnd && i < messages.length; i++) {
2098
2778
  const msg = messages[i];
2099
2779
  const tokens = msgCap > 0
2100
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2101
- : store.estimateTokens(msg);
2780
+ ? Math.min(pse[i], msgCap + 50)
2781
+ : pse[i];
2102
2782
  pickerChunks.push({
2103
2783
  id: msg.id,
2104
2784
  sequence: i,
@@ -2112,8 +2792,8 @@ export class AutobiographicalStrategy {
2112
2792
  for (let i = effectiveRecentStart; i < messages.length; i++) {
2113
2793
  const msg = messages[i];
2114
2794
  const tokens = msgCap > 0
2115
- ? Math.min(store.estimateTokens(msg), msgCap + 50)
2116
- : store.estimateTokens(msg);
2795
+ ? Math.min(pse[i], msgCap + 50)
2796
+ : pse[i];
2117
2797
  pickerChunks.push({
2118
2798
  id: msg.id,
2119
2799
  sequence: i,
@@ -2160,6 +2840,14 @@ export class AutobiographicalStrategy {
2160
2840
  };
2161
2841
  const picker = this.buildPicker(pickerInputs);
2162
2842
  const result = picker.run(pickerInputs, foldingBudget);
2843
+ // Every trust-region override is loud (design §13.4) — silence was half
2844
+ // of the 2026-07-12 incident.
2845
+ const plan = this._lastKvStable?.lastPlan();
2846
+ if (plan?.override) {
2847
+ console.error(`[kv-escalation] override=${plan.override} perturbation=${plan.perturbation}` +
2848
+ ` tokens=${plan.tokens} budget=${foldingBudget.totalBudget}` +
2849
+ ` (see adaptive-resolution-design.md §13.4)`);
2850
+ }
2163
2851
  // Commit the new resolutions back to strategy state for next compile.
2164
2852
  // Persist to chronicle only if anything actually changed — avoids
2165
2853
  // unnecessary state-slot writes on no-op compiles (which is the common
@@ -2196,14 +2884,12 @@ export class AutobiographicalStrategy {
2196
2884
  if (result.produced.length > 0) {
2197
2885
  this.handleProducedOps(result.produced);
2198
2886
  }
2199
- // Hard-fail check: if the picker exhausted itself but the final render
2200
- // would still exceed the HARD budget (not just the soft target), surface
2201
- // an OverBudgetError to the host rather than silently dropping entries.
2202
- // The strategy has done all it can; the application has to decide what
2203
- // to do next (raise budget, switch model, drop windows, etc.).
2204
- if (result.exhausted && result.finalTokens > totalBudget) {
2887
+ // Hard-fail whenever the picker's current plan exceeds the hard budget.
2888
+ // A `produce` op only schedules a missing summary; it does not make the
2889
+ // current raw plan feasible and must never authorize an inference.
2890
+ if (result.finalTokens > rejectionBudget) {
2205
2891
  throw new OverBudgetError({
2206
- budget: totalBudget,
2892
+ budget: rejectionBudget,
2207
2893
  actual: result.finalTokens,
2208
2894
  diagnostics: {
2209
2895
  headTokens,
@@ -2254,7 +2940,7 @@ export class AutobiographicalStrategy {
2254
2940
  // (`maxMessageTokens` is for per-message caps on chat / tool
2255
2941
  // results, not for sharded bodyGroup composites.)
2256
2942
  const tokens = this.estimateTokens(content);
2257
- if (totalTokens + tokens > maxTokens) {
2943
+ if (totalTokens + tokens > prefixBudget) {
2258
2944
  currentRun = null;
2259
2945
  return false;
2260
2946
  }
@@ -2287,7 +2973,7 @@ export class AutobiographicalStrategy {
2287
2973
  sourceRelation: 'derived',
2288
2974
  };
2289
2975
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2290
- if (totalTokens + pairTokens > maxTokens) {
2976
+ if (totalTokens + pairTokens > prefixBudget) {
2291
2977
  currentRun = null;
2292
2978
  return false;
2293
2979
  }
@@ -2357,8 +3043,8 @@ export class AutobiographicalStrategy {
2357
3043
  const resolution = result.finalResolutions.get(msg.id) ?? 0;
2358
3044
  if (resolution === 0) {
2359
3045
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2360
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2361
- if (totalTokens + tokens > maxTokens)
3046
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
3047
+ if (totalTokens + tokens > prefixBudget)
2362
3048
  break;
2363
3049
  entries.push({
2364
3050
  index: entries.length,
@@ -2375,8 +3061,8 @@ export class AutobiographicalStrategy {
2375
3061
  const ancestor = this.findAncestorAt(msg.id, resolution, chunksByMessageId, summariesById);
2376
3062
  if (!ancestor) {
2377
3063
  const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
2378
- const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
2379
- if (totalTokens + tokens > maxTokens)
3064
+ const tokens = msgCap > 0 ? Math.min(pse[i], msgCap + 50) : pse[i];
3065
+ if (totalTokens + tokens > prefixBudget)
2380
3066
  break;
2381
3067
  entries.push({
2382
3068
  index: entries.length,
@@ -2409,7 +3095,7 @@ export class AutobiographicalStrategy {
2409
3095
  sourceRelation: 'derived',
2410
3096
  };
2411
3097
  const pairTokens = this.estimateTokens(questionEntry.content) + this.estimateTokens(answerEntry.content);
2412
- if (totalTokens + pairTokens > maxTokens)
3098
+ if (totalTokens + pairTokens > prefixBudget)
2413
3099
  break;
2414
3100
  entries.push(questionEntry);
2415
3101
  entries.push(answerEntry);
@@ -2418,8 +3104,21 @@ export class AutobiographicalStrategy {
2418
3104
  i++;
2419
3105
  }
2420
3106
  }
2421
- // ----- 6. Emit tail entries newest-first eviction (matches existing behavior) -----
2422
- const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
3107
+ // ----- 6. Emit the fully-reserved tail -----
3108
+ const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, rejectionBudget, totalTokens, pse);
3109
+ if (tailStats.messages !== messages.length - effectiveRecentStart) {
3110
+ throw new OverBudgetError({
3111
+ budget: rejectionBudget,
3112
+ actual: totalTokens + tailTokens,
3113
+ diagnostics: {
3114
+ headTokens,
3115
+ tailTokens,
3116
+ middleTokens: totalTokens - headTokens,
3117
+ middleChunkCount: pickerChunks.length - headMessageIds.size - tailMessageIds.size,
3118
+ deepestLevel,
3119
+ },
3120
+ });
3121
+ }
2423
3122
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
2424
3123
  // ----- 7. Post-process: merge consecutive raw entries from the same bodyGroup -----
2425
3124
  // Both head and tail emission paths emit shards as separate ContextEntries.
@@ -2430,9 +3129,40 @@ export class AutobiographicalStrategy {
2430
3129
  const merged = this.mergeAdjacentBodyGroupRaw(entries, store);
2431
3130
  this.pruneToolEntries(merged);
2432
3131
  this.trimOrphanedToolUse(merged);
3132
+ // Full pairing invariant over the final rendered context — catches the
3133
+ // mid-list orphans the trailing/leading trims can't (bug 6.7). The adaptive
3134
+ // path has the same mid-list orphan producers as hierarchical (budget
3135
+ // `break`s between pair members, raw emission interleaved with recall
3136
+ // pairs), and FKM defaults autobiographical strategies onto this path — so
3137
+ // the guard has to run here too. It's a no-op on already-valid output.
3138
+ this.enforceToolPairing(merged);
2433
3139
  // Strip stale images BEFORE placing markers and committing stats, so both
2434
3140
  // describe the post-strip context the agent actually receives.
2435
3141
  this.applyImageStripping(merged, store);
3142
+ // The newest stored message is the triggering turn for this compile. A
3143
+ // structural repair may rewrite tool blocks, but it must never erase that
3144
+ // turn. Body-group shards merge under the first shard's source id, so any
3145
+ // surviving member of the same group proves the newest shard is present.
3146
+ const newest = messages[messages.length - 1];
3147
+ if (newest) {
3148
+ const newestGroupIds = newest.bodyGroupId
3149
+ ? new Set(messages.filter(m => m.bodyGroupId === newest.bodyGroupId).map(m => m.id))
3150
+ : new Set([newest.id]);
3151
+ const newestRetained = merged.some(entry => entry.sourceMessageId && newestGroupIds.has(entry.sourceMessageId));
3152
+ if (!newestRetained) {
3153
+ throw new OverBudgetError({
3154
+ budget: rejectionBudget,
3155
+ actual: totalTokens + tailTokens,
3156
+ diagnostics: {
3157
+ headTokens,
3158
+ tailTokens,
3159
+ middleTokens: totalTokens - headTokens,
3160
+ middleChunkCount: pickerChunks.length - headMessageIds.size - tailMessageIds.size,
3161
+ deepestLevel,
3162
+ },
3163
+ });
3164
+ }
3165
+ }
2436
3166
  // Place ≤4 cache breakpoints across the FINAL ordered entries so the
2437
3167
  // provider can reuse the stable folded prefix — not just the head. With a
2438
3168
  // single head marker the cache hit is ~2%; well-placed breakpoints take the
@@ -2440,8 +3170,91 @@ export class AutobiographicalStrategy {
2440
3170
  // placement is the dominant KV lever).
2441
3171
  this.placeCacheMarkers(merged, headMessageIds, tailMessageIds);
2442
3172
  this.rsEnd();
3173
+ // Closed-loop calibration bookkeeping: the committed render stats total
3174
+ // (in CURRENT calibrated units) is what this compile claims the request
3175
+ // will cost — reportRealInputTokens compares provider usage against it.
3176
+ this._storeView = store;
3177
+ const rs = this.getRenderStats(store);
3178
+ this._lastCompileEstimate =
3179
+ rs.head.tokens + rs.tail.tokens + rs.middleRaw.tokens +
3180
+ rs.summaries.l1.tokens + rs.summaries.l2.tokens + rs.summaries.l3.tokens;
3181
+ this._calibrationArmed = true; // exactly one sample per compile
2443
3182
  return merged;
2444
3183
  }
3184
+ _lastCompileEstimate = 0;
3185
+ _storeView = null;
3186
+ /**
3187
+ * Closed-loop estimator calibration (2026-07-12). Feed the REAL input total
3188
+ * for a request built from the latest compile (fresh + cache_read +
3189
+ * cache_creation, minus non-window overhead the caller knows about, e.g.
3190
+ * tool schemas). Maintains an EMA of real/estimated and applies it as the
3191
+ * store's global multiplier, persisted across restarts. The per-class rates
3192
+ * carry the shape; this carries the residual level.
3193
+ */
3194
+ reportRealInputTokens(realTotal) {
3195
+ if (!Number.isFinite(realTotal) || realTotal <= 0)
3196
+ return;
3197
+ const est = this._lastCompileEstimate;
3198
+ if (!est || est <= 0)
3199
+ return;
3200
+ // ARM-ONCE-PER-COMPILE (2026-07-12 regression fix). A turn makes MANY API
3201
+ // calls — tool-use rounds and max_tokens continuations each append to the
3202
+ // request — so only the FIRST completion after a compile was built from
3203
+ // the window this estimate describes. Feeding later calls compares a grown
3204
+ // request against the original estimate: ratios of 2.0-2.3 (est=224k
3205
+ // real=504k) drove the multiplier 1.0 -> 2.37 in minutes, inflating every
3206
+ // estimate (the fold floor went 62k -> 108k on unchanged content) and
3207
+ // exhausting the picker on every wake. One sample per compile, always.
3208
+ if (!this._calibrationArmed)
3209
+ return;
3210
+ this._calibrationArmed = false;
3211
+ const ratio = realTotal / est;
3212
+ // SANITY BAND: a representative sample sits near 1. Anything wilder is a
3213
+ // structural mismatch (a request we didn't compile, a partial compile, a
3214
+ // provider quirk) — never evidence about chars-per-token. Log, don't learn.
3215
+ if (ratio < 0.6 || ratio > 1.8) {
3216
+ console.error(`[estimator-calibration] REJECTED out-of-band sample real/est=${ratio.toFixed(2)} ` +
3217
+ `(est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k) — ` +
3218
+ `not a window-shaped request; multiplier stays ${this._calibration.toFixed(2)}`);
3219
+ return;
3220
+ }
3221
+ const current = this._calibration;
3222
+ const observed = ratio * current; // back out the multiplier already applied
3223
+ const alpha = 0.2; // slow EMA: one wild request shouldn't yank the ruler
3224
+ const next = current + alpha * (observed - current);
3225
+ const clamped = Math.min(1.8, Math.max(0.6, next));
3226
+ if (Math.abs(clamped - current) / current > 0.02) {
3227
+ console.error(`[estimator-calibration] real/est=${ratio.toFixed(2)} ` +
3228
+ `multiplier ${current.toFixed(2)} -> ${clamped.toFixed(2)} (est=${Math.round(est / 1000)}k real=${Math.round(realTotal / 1000)}k)`);
3229
+ }
3230
+ this._calibration = clamped;
3231
+ this.applyCalibration();
3232
+ try {
3233
+ this.store?.setStateJson(this.calibrationStateId, { multiplier: this._calibration, at: Date.now() });
3234
+ }
3235
+ catch { /* persistence is best-effort */ }
3236
+ }
3237
+ _calibrationArmed = false;
3238
+ _calibration = 1;
3239
+ _calibrationLoaded = false;
3240
+ applyCalibration() {
3241
+ this._storeView?.setTokenCalibration?.(this._calibration);
3242
+ }
3243
+ /** Load the persisted multiplier once and push it into the store view. */
3244
+ loadCalibration(store) {
3245
+ this._storeView = store;
3246
+ if (!this._calibrationLoaded) {
3247
+ this._calibrationLoaded = true;
3248
+ try {
3249
+ const saved = this.store?.getStateJson(this.calibrationStateId);
3250
+ if (saved && Number.isFinite(saved.multiplier)) {
3251
+ this._calibration = Math.min(1.8, Math.max(0.6, saved.multiplier));
3252
+ }
3253
+ }
3254
+ catch { /* absent slot is fine */ }
3255
+ }
3256
+ this.applyCalibration();
3257
+ }
2445
3258
  /**
2446
3259
  * Place up to four `cache_control` breakpoints across the final ordered
2447
3260
  * entries: the head/system boundary, the end of the folded history (the
@@ -2496,13 +3309,23 @@ export class AutobiographicalStrategy {
2496
3309
  mergeAdjacentBodyGroupRaw(entries, store) {
2497
3310
  if (entries.length === 0)
2498
3311
  return entries;
2499
- // Look up bodyGroupId by sourceMessageId via the message store.
2500
- const groupOf = (sourceMessageId) => {
3312
+ // Look up bodyGroup metadata by sourceMessageId, memoized for the
3313
+ // duration of this pass. Entries reference the same messages repeatedly
3314
+ // (run-extension checks + the shardIndex sort comparator below), and
3315
+ // store.get() also resolves blobs — fetch each message at most once.
3316
+ const shardMeta = new Map();
3317
+ const metaOf = (sourceMessageId) => {
2501
3318
  if (!sourceMessageId)
2502
3319
  return undefined;
2503
- const m = store.get(sourceMessageId);
2504
- return m?.bodyGroupId;
3320
+ let meta = shardMeta.get(sourceMessageId);
3321
+ if (!meta) {
3322
+ const m = store.get(sourceMessageId);
3323
+ meta = { groupId: m?.bodyGroupId, shardIndex: m?.shardIndex };
3324
+ shardMeta.set(sourceMessageId, meta);
3325
+ }
3326
+ return meta;
2505
3327
  };
3328
+ const groupOf = (sourceMessageId) => metaOf(sourceMessageId)?.groupId;
2506
3329
  const out = [];
2507
3330
  let i = 0;
2508
3331
  while (i < entries.length) {
@@ -2531,9 +3354,9 @@ export class AutobiographicalStrategy {
2531
3354
  // ordering. (Head/tail emission keeps chronological order, but defending
2532
3355
  // against reorderings is cheap.)
2533
3356
  const sortedRun = [...run].sort((a, b) => {
2534
- const ma = a.sourceMessageId ? store.get(a.sourceMessageId) : null;
2535
- const mb = b.sourceMessageId ? store.get(b.sourceMessageId) : null;
2536
- return (ma?.shardIndex ?? 0) - (mb?.shardIndex ?? 0);
3357
+ const sa = metaOf(a.sourceMessageId)?.shardIndex ?? 0;
3358
+ const sb = metaOf(b.sourceMessageId)?.shardIndex ?? 0;
3359
+ return sa - sb;
2537
3360
  });
2538
3361
  // Build merged text content. Non-text blocks (rare in shards) are
2539
3362
  // preserved on the first shard's entry only.
@@ -2579,13 +3402,83 @@ export class AutobiographicalStrategy {
2579
3402
  */
2580
3403
  buildPicker(inputs) {
2581
3404
  if (this.config.foldingStrategy === 'kv-stable') {
2582
- return new Picker(new KvStableStrategy(inputs, {
3405
+ const strategy = new KvStableStrategy(inputs, {
2583
3406
  reachTokens: this.config.kvStableReachTokens,
3407
+ qualityGapRatio: this.config.kvStableQualityGapRatio,
2584
3408
  mergeThreshold: this.config.mergeThreshold,
2585
- }));
3409
+ });
3410
+ this._lastKvStable = strategy;
3411
+ return new Picker(strategy);
2586
3412
  }
3413
+ this._lastKvStable = null;
2587
3414
  return this.getAdaptivePicker();
2588
3415
  }
3416
+ /** The kv-stable strategy instance behind the most recent compile — kept for
3417
+ * `[kv-escalation]` observability (design §13.4: every override is loud). */
3418
+ _lastKvStable = null;
3419
+ /**
3420
+ * Static salience prior (design §13.3) — "is the window the only copy?".
3421
+ * Content whose payload is externalized folds cheap: tool blocks
3422
+ * (re-derivable), fenced code (usually written to disk), images (the file/
3423
+ * CDN keeps them), bare link drops. Conversation exists nowhere but the
3424
+ * chronicle, so it stays at 1. Returns a value in [0.2, 1]; cheap,
3425
+ * deterministic, computed per message at picker-input construction.
3426
+ */
3427
+ static staticSalience(msg) {
3428
+ let totalChars = 0;
3429
+ let externalChars = 0;
3430
+ for (const block of msg.content) {
3431
+ const b = block;
3432
+ switch (b.type) {
3433
+ case 'text': {
3434
+ const t = b.text ?? '';
3435
+ totalChars += t.length;
3436
+ // Fenced code blocks.
3437
+ const fences = t.split('```');
3438
+ for (let i = 1; i < fences.length; i += 2)
3439
+ externalChars += fences[i].length;
3440
+ // Bare link-drop lines (the URL is the payload).
3441
+ for (const line of t.split('\n')) {
3442
+ const trimmed = line.trim();
3443
+ if (/^https?:\/\/\S+$/.test(trimmed))
3444
+ externalChars += trimmed.length;
3445
+ }
3446
+ break;
3447
+ }
3448
+ case 'tool_use': {
3449
+ const n = JSON.stringify(b.input ?? {}).length;
3450
+ totalChars += n;
3451
+ externalChars += n;
3452
+ break;
3453
+ }
3454
+ case 'tool_result': {
3455
+ const n = typeof b.content === 'string'
3456
+ ? b.content.length
3457
+ : JSON.stringify(b.content ?? '').length;
3458
+ totalChars += n;
3459
+ externalChars += n;
3460
+ break;
3461
+ }
3462
+ case 'image': {
3463
+ // Estimate parity with the renderer's flat image cost; the payload
3464
+ // lives in the file/CDN, so it is fully externalized.
3465
+ totalChars += 6400; // ≈1600 tokens × 4 chars
3466
+ externalChars += 6400;
3467
+ break;
3468
+ }
3469
+ default: {
3470
+ const t = b.text ?? '';
3471
+ totalChars += t.length;
3472
+ }
3473
+ }
3474
+ }
3475
+ if (totalChars <= 0)
3476
+ return 1;
3477
+ const externalized = Math.min(1, externalChars / totalChars);
3478
+ // Fully-externalized content bottoms out at 0.2 — cheap, never free
3479
+ // (hard protections, not salience, are what make content unfoldable).
3480
+ return Math.max(0.2, 1 - 0.8 * externalized);
3481
+ }
2589
3482
  /**
2590
3483
  * Walk the summary tree to find the L_k ancestor of a message.
2591
3484
  * Returns null if no ancestor exists at that level (e.g., L_k not yet produced).
@@ -2735,6 +3628,85 @@ export class AutobiographicalStrategy {
2735
3628
  const { selected: l1Selected, tokensUsed: l1Used } = this.selectL1Summaries(shownL1, l1Effective, l1Remaining);
2736
3629
  selectedSummaries.push(...l1Selected);
2737
3630
  totalSummaryTokens += l1Used;
3631
+ // Phase 3b: coverage repair (bug 6.9). getAntiRedundantSummaries excluded
3632
+ // an L2 (or L3) when ALL of its children were in the CANDIDATE shown-set —
3633
+ // computed before budget selection. If the budget then dropped some of
3634
+ // those children (e.g. KnowledgeStrategy's research L1 cap), the covered
3635
+ // history appears at NEITHER level: a silent memory hole. Re-include any
3636
+ // excluded L2/L3 whose children did not all make the final selection.
3637
+ // Some overlap with the children that DID survive is accepted — coverage
3638
+ // beats perfect dedup here.
3639
+ //
3640
+ // Repairs are additionally bounded by a per-level ALLOWANCE (a fraction of
3641
+ // that level's budget), not just the overall context budget. The
3642
+ // excluded-with-partially-dropped-children state only arises from a store
3643
+ // damaged mid-merge (the crash window at compressChunkHierarchical, or the
3644
+ // legacy setMergedInto index-desync). On such a store MANY L2s can be in
3645
+ // this state at once; without a cap, re-including all of them at full size
3646
+ // would starve the recent window via Phase 4's newest-first eviction. When
3647
+ // repairs exceed the allowance we stop re-including and warn — a corrupted
3648
+ // store announces itself instead of silently trading recent messages for
3649
+ // redundant summaries.
3650
+ {
3651
+ // Allowance = a fraction of the level budget, with a floor tied to the
3652
+ // overall budget so a strategy that zeroes a level budget (e.g.
3653
+ // KnowledgeStrategy prioritising L1) can still repair a handful of
3654
+ // covering summaries, while a corrupted store with dozens of them stays
3655
+ // bounded well short of the recent window.
3656
+ const REPAIR_ALLOWANCE_FRACTION = 0.25;
3657
+ const REPAIR_FLOOR_FRACTION = 0.05;
3658
+ const repairFloor = maxTokens * REPAIR_FLOOR_FRACTION;
3659
+ const l2RepairAllowance = Math.max(l2Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
3660
+ const l3RepairAllowance = Math.max(l3Budget * REPAIR_ALLOWANCE_FRACTION, repairFloor);
3661
+ const selectedIds = new Set(selectedSummaries.map(s => s.id));
3662
+ const shownL2Ids = new Set(shownL2.map(s => s.id));
3663
+ const shownL3Ids = new Set(shownL3.map(s => s.id));
3664
+ let l2RepairTokens = 0;
3665
+ let l3RepairTokens = 0;
3666
+ let l2RepairsSkipped = 0;
3667
+ let l3RepairsSkipped = 0;
3668
+ // L2s excluded by anti-redundancy: unmerged, not in shownL2.
3669
+ for (const s of this.summaries) {
3670
+ if (s.level !== 2 || s.mergedInto || shownL2Ids.has(s.id))
3671
+ continue;
3672
+ if (s.sourceIds.every(id => selectedIds.has(id)))
3673
+ continue; // truly redundant
3674
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
3675
+ continue;
3676
+ if (l2RepairTokens + s.tokens > l2RepairAllowance) {
3677
+ l2RepairsSkipped++;
3678
+ continue;
3679
+ }
3680
+ selectedSummaries.push(s);
3681
+ totalSummaryTokens += s.tokens;
3682
+ l2RepairTokens += s.tokens;
3683
+ selectedIds.add(s.id);
3684
+ }
3685
+ // L3s excluded by anti-redundancy — after L2 repair, so a repaired L2
3686
+ // counts as selected coverage for its parent L3.
3687
+ for (const s of this.summaries) {
3688
+ if (s.level !== 3 || s.mergedInto || shownL3Ids.has(s.id))
3689
+ continue;
3690
+ if (s.sourceIds.every(id => selectedIds.has(id)))
3691
+ continue;
3692
+ if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
3693
+ continue;
3694
+ if (l3RepairTokens + s.tokens > l3RepairAllowance) {
3695
+ l3RepairsSkipped++;
3696
+ continue;
3697
+ }
3698
+ selectedSummaries.push(s);
3699
+ totalSummaryTokens += s.tokens;
3700
+ l3RepairTokens += s.tokens;
3701
+ selectedIds.add(s.id);
3702
+ }
3703
+ if (l2RepairsSkipped > 0 || l3RepairsSkipped > 0) {
3704
+ console.warn(`[AutobiographicalStrategy] coverage-repair allowance exceeded — ` +
3705
+ `skipped ${l2RepairsSkipped} L2 and ${l3RepairsSkipped} L3 re-inclusions ` +
3706
+ `(store likely corrupted mid-merge). Some covered history may render at ` +
3707
+ `no summary level this pass.`);
3708
+ }
3709
+ }
2738
3710
  // Emit summaries + pinned messages between head and recent windows.
2739
3711
  //
2740
3712
  // Default (positionedRecallPairs=true): one Q/A pair per summary,
@@ -2812,6 +3784,11 @@ export class AutobiographicalStrategy {
2812
3784
  for (const item of items) {
2813
3785
  if (item.kind === 'summary') {
2814
3786
  const summary = item.summary;
3787
+ // Defensive: never emit a recall pair for an empty/bugged summary — an
3788
+ // empty assistant text block triggers a 400. (Production guards too,
3789
+ // but a legacy empty summary may already exist in the store.)
3790
+ if (!summary.content || !summary.content.trim())
3791
+ continue;
2815
3792
  const headerText = this.buildRecallHeader(summary);
2816
3793
  const questionEntry = {
2817
3794
  index: entries.length,
@@ -2916,6 +3893,9 @@ export class AutobiographicalStrategy {
2916
3893
  const tailStats = this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
2917
3894
  this.rsRaw('tail', tailStats.tokens, tailStats.messages);
2918
3895
  this.trimOrphanedToolUse(entries);
3896
+ // Full pairing invariant over the final rendered context — catches the
3897
+ // mid-list orphans the trailing/leading trims can't (bug 6.7).
3898
+ this.enforceToolPairing(entries);
2919
3899
  this.pruneToolEntries(entries);
2920
3900
  // Strip stale images before committing stats so RenderStats.total reflects
2921
3901
  // the post-strip context (this path places no cache markers).
@@ -3189,24 +4169,81 @@ export class AutobiographicalStrategy {
3189
4169
  return out;
3190
4170
  }
3191
4171
  /**
3192
- * Rebuild chunk boundaries based on current messages.
4172
+ * Rebuild the chunk list: persisted records own the past; the running-sum
4173
+ * chunker only extends at the frontier, and a chunk is only ever created
4174
+ * once it CLOSES (reaches targetChunkTokens). The trailing partial chunk
4175
+ * is never created and never compressed — eager partial-tail compression
4176
+ * minted a new near-duplicate L1 per rebuild while the tail grew (the
4177
+ * prefix-generation families found fleet-wide in the 2026-07 audit).
3193
4178
  */
3194
4179
  rebuildChunks(store) {
3195
- const messagesToChunk = this.getCompressibleMessages(store);
3196
- // Preserve existing compressed chunks (legacy) and summary linkage (hierarchical)
3197
- const existingCompressed = new Map();
4180
+ this.chunks = [];
4181
+ this.compressionQueue = [];
4182
+ // ---- 1. Materialize persisted records (they OWN their messages). ----
4183
+ const byId = new Map();
4184
+ for (const m of store.getAll())
4185
+ byId.set(m.id, m);
4186
+ const consumed = new Set();
4187
+ let orphaned = 0;
4188
+ for (const rec of this.chunkRecords) {
4189
+ const msgs = [];
4190
+ for (const id of rec.sourceIds) {
4191
+ const m = byId.get(id);
4192
+ if (m)
4193
+ msgs.push(m);
4194
+ }
4195
+ if (msgs.length === 0) {
4196
+ orphaned++;
4197
+ continue;
4198
+ }
4199
+ for (const m of msgs)
4200
+ consumed.add(m.id);
4201
+ const chunk = {
4202
+ index: this.chunks.length,
4203
+ startIndex: -1, // record-derived; filtered-array indices are not meaningful
4204
+ endIndex: -1,
4205
+ messages: msgs,
4206
+ tokens: msgs.reduce((sum, m) => sum + (this.config.attachmentsIgnoreSize
4207
+ ? this.estimateTextOnlyTokens(m)
4208
+ : store.estimateTokens(m)), 0),
4209
+ compressed: rec.compressed,
4210
+ summaryId: rec.summaryId,
4211
+ phaseType: rec.phaseType,
4212
+ recordId: rec.id,
4213
+ };
4214
+ this.chunks.push(chunk);
4215
+ }
4216
+ // ---- 2. Fail closed on the chain-break signature. ----
4217
+ // Most records resolving to zero live messages means message identity
4218
+ // has been rebuilt/renumbered underneath us. Chunking "fresh" ground now
4219
+ // would re-compress already-lived history into duplicate memories.
4220
+ // Halt ALL compression until an operator reconciles the store.
4221
+ if (this.chunkPersistenceEnabled && this.chunkRecords.length >= 3 &&
4222
+ orphaned / this.chunkRecords.length > 0.5) {
4223
+ this.chunkRecordsOrphaned = true;
4224
+ this.compressionQueue = [];
4225
+ if (!this._orphanWarned) {
4226
+ this._orphanWarned = true;
4227
+ console.error(`[autobiographical] FAIL-CLOSED: ${orphaned}/${this.chunkRecords.length} chunk ` +
4228
+ `records resolve to zero live messages (messages chain break / store ` +
4229
+ `reconciliation signature). Compression halted to prevent duplicate ` +
4230
+ `memory formation — reconcile the store before resuming.`);
4231
+ }
4232
+ return;
4233
+ }
4234
+ this.chunkRecordsOrphaned = false;
4235
+ // Queue uncompressed record-backed chunks (crash-recovery: record was
4236
+ // appended but the process died before its L1 landed).
3198
4237
  for (const chunk of this.chunks) {
3199
- if (chunk.compressed) {
3200
- const key = this.chunkKey(chunk);
3201
- existingCompressed.set(key, chunk);
4238
+ if (!chunk.compressed && !(chunk.recordId && this._overlapBlocked.has(chunk.recordId))) {
4239
+ this.compressionQueue.push(chunk.index);
3202
4240
  }
3203
4241
  }
3204
- // Rebuild chunks
3205
- this.chunks = [];
3206
- this.compressionQueue = [];
4242
+ // ---- 3. Chunk the frontier: compressible messages not owned by any record. ----
4243
+ const messagesToChunk = this.getCompressibleMessages(store)
4244
+ .filter(m => !consumed.has(m.id));
3207
4245
  let currentChunk = [];
3208
4246
  let currentTokens = 0;
3209
- // Track start position in the filtered array for chunk boundary metadata
3210
4247
  let chunkFilteredStart = 0;
3211
4248
  for (let i = 0; i < messagesToChunk.length; i++) {
3212
4249
  const msg = messagesToChunk[i];
@@ -3229,22 +4266,53 @@ export class AutobiographicalStrategy {
3229
4266
  continue;
3230
4267
  }
3231
4268
  if (shouldClose) {
3232
- const chunk = this.createChunk(this.chunks.length, chunkFilteredStart, i + 1, currentChunk, currentTokens, existingCompressed);
3233
- this.chunks.push(chunk);
3234
- if (!chunk.compressed) {
3235
- this.compressionQueue.push(chunk.index);
4269
+ const chunk = {
4270
+ index: this.chunks.length,
4271
+ startIndex: chunkFilteredStart,
4272
+ endIndex: i + 1,
4273
+ messages: [...currentChunk],
4274
+ tokens: currentTokens,
4275
+ compressed: false,
4276
+ };
4277
+ // Persist the boundary the moment it closes — from here on this
4278
+ // span is owned and never re-keyed by config drift or restarts.
4279
+ if (this.chunkPersistenceEnabled) {
4280
+ const record = {
4281
+ id: `c-${this.chunkIdCounter++}`,
4282
+ sourceIds: chunk.messages.map(m => m.id),
4283
+ compressed: false,
4284
+ };
4285
+ this.appendChunkRecord(record);
4286
+ chunk.recordId = record.id;
3236
4287
  }
4288
+ this.chunks.push(chunk);
4289
+ this.compressionQueue.push(chunk.index);
3237
4290
  currentChunk = [];
3238
4291
  currentTokens = 0;
3239
4292
  chunkFilteredStart = i + 1;
3240
4293
  }
3241
4294
  }
3242
- if (currentChunk.length >= 4) {
3243
- const chunk = this.createChunk(this.chunks.length, chunkFilteredStart, messagesToChunk.length, currentChunk, currentTokens, existingCompressed);
3244
- this.chunks.push(chunk);
3245
- if (!chunk.compressed) {
3246
- this.compressionQueue.push(chunk.index);
3247
- }
4295
+ // NOTE: no trailing-partial chunk. An unclosed chunk is not a chunk —
4296
+ // it compresses only after the running sum closes it.
4297
+ // ---- 4. L1 holdback: keep the newest X closed chunks out of the
4298
+ // speculative queue (default 1). The chunk at the live edge is the one
4299
+ // most likely to still be in motion (edits, tool-result landings, the
4300
+ // episode it belongs to still resolving); summarize it once a newer chunk
4301
+ // has closed behind it. The queue is rebuilt on every message, so a
4302
+ // held-back chunk is released automatically the moment it ages out of the
4303
+ // window. Demand overrides: a picker `produce` op (enqueueL1ForRange)
4304
+ // marks the chunk demanded, and demanded chunks are never held back —
4305
+ // when folding actually NEEDS the L1, production must not be blocked.
4306
+ const holdback = this.config.l1HoldbackChunks ?? 1;
4307
+ if (holdback > 0 && this.chunks.length > 0) {
4308
+ const cutoff = this.chunks.length - holdback;
4309
+ this.compressionQueue = this.compressionQueue.filter((idx) => {
4310
+ if (idx < cutoff)
4311
+ return true;
4312
+ const ch = this.chunks[idx];
4313
+ const lastId = ch?.messages[ch.messages.length - 1]?.id;
4314
+ return lastId !== undefined && this._demandedL1Chunks.has(lastId);
4315
+ });
3248
4316
  }
3249
4317
  }
3250
4318
  /**
@@ -3325,7 +4393,8 @@ export class AutobiographicalStrategy {
3325
4393
  applyImageStripping(entries, store) {
3326
4394
  const maxLive = this.config.maxLiveImages ?? 0; // 0 = unlimited count
3327
4395
  const depthTokens = this.config.imageStripDepthTokens ?? 0; // 0 = no depth strip
3328
- if (maxLive === 0 && depthTokens === 0)
4396
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4397
+ if (maxLive === 0 && depthTokens === 0 && maxLiveBytes === 0)
3329
4398
  return; // policy disabled
3330
4399
  const messages = store.getAll();
3331
4400
  const posById = new Map();
@@ -3357,6 +4426,7 @@ export class AutobiographicalStrategy {
3357
4426
  .filter(({ idx }) => this.hasImageBlock(entries[idx].content))
3358
4427
  .sort((a, b) => b.pos - a.pos);
3359
4428
  let keptImages = 0;
4429
+ let keptImageBytes = 0;
3360
4430
  for (const { idx, pos } of ordered) {
3361
4431
  const entry = entries[idx];
3362
4432
  const tooDeep = depthTokens > 0 && (pos < 0 || pos < stripStart);
@@ -3364,26 +4434,136 @@ export class AutobiographicalStrategy {
3364
4434
  entry.content = entry.content.map((block) => {
3365
4435
  if (block.type !== 'image')
3366
4436
  return block;
4437
+ const blockBytes = AutobiographicalStrategy.imageBlockBytes(block);
3367
4438
  const overCount = maxLive > 0 && keptImages >= maxLive;
3368
- if (tooDeep || overCount) {
3369
- // The renderer estimates an image at `tokenEstimate ?? 1600` (see
3370
- // message-store/context-log). Hand that back to the bucket, less the
3371
- // placeholder text that replaces it, so the stats match the output.
3372
- const reclaimed = (block.tokenEstimate ?? 1600) - placeholderTokens;
3373
- if (this._rs && reclaimed > 0)
3374
- this._rs[bucket].tokens -= reclaimed;
4439
+ const overBytes = maxLiveBytes > 0 && keptImageBytes + blockBytes > maxLiveBytes;
4440
+ if (tooDeep || overCount || overBytes) {
4441
+ // Stats-neutral (2026-07-12): every budgeting site now tallies at
4442
+ // POST-STRIP prices (see postStripEstimates), so the bucket never
4443
+ // charged this image at full weight — reclaiming here would
4444
+ // double-decrement. The strip pass only swaps the block.
4445
+ void bucket;
4446
+ void placeholderTokens;
3375
4447
  return { type: 'text', text: AutobiographicalStrategy.IMAGE_PLACEHOLDER };
3376
4448
  }
3377
4449
  keptImages++;
4450
+ keptImageBytes += blockBytes;
3378
4451
  return block;
3379
4452
  });
3380
4453
  }
3381
4454
  }
4455
+ /**
4456
+ * Post-strip token estimate per message index (2026-07-12 tail-starvation
4457
+ * fix). Mirrors `applyImageStripping`: an image beyond the `maxLiveImages`
4458
+ * newest (counted newest-first) or deeper than `imageStripDepthTokens` of
4459
+ * raw estimate from the live end renders as a placeholder — so every place
4460
+ * that BUDGETS messages (recent-window walk-back, head/tail sums, middle
4461
+ * chunk sizes) must cost it as one. Pricing stripped images at their full
4462
+ * estimate collapsed an image-dense tail to a fraction of its configured
4463
+ * size (42k rendered of a 120k window), and pricing them post-strip in the
4464
+ * walk-back alone made the picker's raw-priced tail overflow the budget
4465
+ * (318k) — the estimate must be consistent EVERYWHERE.
4466
+ */
4467
+ postStripEstimates(store) {
4468
+ const messages = store.getAll();
4469
+ const out = new Array(messages.length);
4470
+ const stripDepth = this.config.imageStripDepthTokens ?? 0;
4471
+ const maxLive = this.config.maxLiveImages ?? 0;
4472
+ const maxLiveBytes = this.config.maxLiveImageBytes ?? AutobiographicalStrategy.DEFAULT_MAX_LIVE_IMAGE_BYTES;
4473
+ const stripActive = stripDepth > 0 || maxLive > 0 || maxLiveBytes > 0;
4474
+ const placeholderTokens = Math.ceil(AutobiographicalStrategy.IMAGE_PLACEHOLDER.length / 4);
4475
+ let liveImagesSeen = 0;
4476
+ let liveImageBytes = 0;
4477
+ let rawDepth = 0; // raw-estimate depth from the newest message (mirrors getImageStripStart)
4478
+ for (let i = messages.length - 1; i >= 0; i--) {
4479
+ const raw = store.estimateTokens(messages[i]);
4480
+ let est = raw;
4481
+ if (stripActive) {
4482
+ for (const b of messages[i].content) {
4483
+ if (b.type !== 'image')
4484
+ continue;
4485
+ const bytes = AutobiographicalStrategy.imageBlockBytes(b);
4486
+ const beyondDepth = stripDepth > 0 && rawDepth > stripDepth;
4487
+ const beyondCount = maxLive > 0 && liveImagesSeen >= maxLive;
4488
+ const beyondBytes = maxLiveBytes > 0 && liveImageBytes + bytes > maxLiveBytes;
4489
+ if (beyondDepth || beyondCount || beyondBytes) {
4490
+ const imgEst = b.tokenEstimate ?? 1600;
4491
+ est -= Math.max(0, imgEst - placeholderTokens);
4492
+ }
4493
+ else {
4494
+ liveImagesSeen++;
4495
+ liveImageBytes += bytes;
4496
+ }
4497
+ }
4498
+ }
4499
+ rawDepth += raw;
4500
+ out[i] = est;
4501
+ }
4502
+ return out;
4503
+ }
4504
+ /** Byte wall default: 20MB of base64 (API total-request cap is 32MB). */
4505
+ static DEFAULT_MAX_LIVE_IMAGE_BYTES = 20 * 1024 * 1024;
4506
+ /** Compression prompts carry head + recall frontier + raw chunk alongside
4507
+ * their images, so they get a tighter image budget than the live window. */
4508
+ static DEFAULT_MAX_COMPRESSION_IMAGE_BYTES = 12 * 1024 * 1024;
4509
+ /** Base64 payload size of an image block (0 for non-base64 sources). */
4510
+ static imageBlockBytes(b) {
4511
+ const src = b.source;
4512
+ return typeof src?.data === 'string' ? src.data.length : 0;
4513
+ }
4514
+ /**
4515
+ * Cap inline image bytes in a COMPRESSION prompt (2026-07-12). The main
4516
+ * window enforces `maxLiveImageBytes` through the strip policy; the
4517
+ * summarizer's raw-chunk replay had no such wall and leaned on membrane's
4518
+ * byte shed instead — which is a transport backstop, not a policy owner
4519
+ * (it fired at 27MB on a live merge). Keep images newest-first within the
4520
+ * budget; older ones become the same loud placeholder the window uses, so
4521
+ * the summarizer is told plainly that it is not seeing them.
4522
+ * Returns the number of images replaced.
4523
+ */
4524
+ capCompressionImageBytes(messages, capBytes) {
4525
+ if (capBytes <= 0)
4526
+ return 0;
4527
+ let kept = 0;
4528
+ let dropped = 0;
4529
+ // Recurse into tool_result content: an agent that drives a shell/plotter/
4530
+ // browser carries most of its image bytes NESTED in tool results, not as
4531
+ // top-level blocks. Capping only the top level left those untouched and
4532
+ // membrane's transport shed kept firing at 27MB (2026-07-12).
4533
+ const capBlocks = (blocks) => blocks.map((b) => {
4534
+ if (b.type === 'image') {
4535
+ const bytes = AutobiographicalStrategy.imageBlockBytes(b);
4536
+ if (kept + bytes <= capBytes) {
4537
+ kept += bytes;
4538
+ return b;
4539
+ }
4540
+ dropped++;
4541
+ return { type: 'text', text: AutobiographicalStrategy.IMAGE_PLACEHOLDER };
4542
+ }
4543
+ const nested = b.content;
4544
+ if (b.type === 'tool_result' && Array.isArray(nested)) {
4545
+ return { ...b, content: capBlocks(nested) };
4546
+ }
4547
+ return b;
4548
+ });
4549
+ for (let i = messages.length - 1; i >= 0; i--) {
4550
+ const m = messages[i];
4551
+ if (!Array.isArray(m.content))
4552
+ continue;
4553
+ m.content = capBlocks(m.content);
4554
+ }
4555
+ if (dropped > 0) {
4556
+ console.error(`[autobiographical] compression prompt: replaced ${dropped} older image(s) with placeholders ` +
4557
+ `to stay under the ${Math.round(capBytes / 1e6)}MB image-byte budget (kept ${Math.round(kept / 1e6)}MB, newest-first)`);
4558
+ }
4559
+ return dropped;
4560
+ }
3382
4561
  getRecentWindowStart(store) {
3383
4562
  const messages = store.getAll();
4563
+ const pse = this.postStripEstimates(store);
3384
4564
  let tokens = 0;
3385
4565
  for (let i = messages.length - 1; i >= 0; i--) {
3386
- tokens += store.estimateTokens(messages[i]);
4566
+ tokens += pse[i];
3387
4567
  if (tokens > this.config.recentWindowTokens) {
3388
4568
  let boundary = i + 1;
3389
4569
  // Don't split a tool_use/tool_result pair: if the message at the boundary
@@ -3464,6 +4644,171 @@ export class AutobiographicalStrategy {
3464
4644
  }
3465
4645
  }
3466
4646
  }
4647
+ /** Placeholder body for a stub tool_result inserted by enforceToolPairing. */
4648
+ static STUB_TOOL_RESULT_TEXT = '[tool result unavailable — omitted during context compression]';
4649
+ /**
4650
+ * Final post-selection tool-pairing validator (bug 6.7).
4651
+ *
4652
+ * The Anthropic API requires every `tool_use` block to be answered by a
4653
+ * matching `tool_result` in the immediately-following message, and every
4654
+ * `tool_result` to answer a `tool_use` in the immediately-preceding
4655
+ * message. Selection can violate this mid-list in ways the trailing
4656
+ * (`trimOrphanedToolUse`) and leading orphan trims don't catch:
4657
+ *
4658
+ * - a budget `break` cutting between a raw pin pair's two messages;
4659
+ * - the uncompressed-chunk fallback emitting a raw tool_result whose
4660
+ * tool_use chunk already compressed (or vice versa);
4661
+ * - a recall pair or pin interleaving between a tool_use and its result.
4662
+ *
4663
+ * Repair policy prefers preserving content over dropping:
4664
+ *
4665
+ * - a tool_use whose result is missing from the next entry first triggers
4666
+ * a short look-ahead: if the genuine (displaced) result is a few entries
4667
+ * down it is MOVED up into position (see
4668
+ * {@link relocateOrDropMissingResults}); only when no real result exists
4669
+ * is a STUB tool_result emitted. Either way the result block is merged
4670
+ * into the next entry when that entry already carries results for this
4671
+ * cycle, or inserted as a new user entry;
4672
+ * - a tool_result whose tool_use is not in the immediately-preceding
4673
+ * entry (and was not relocated) is dropped — there is no safe way to
4674
+ * stub a tool_use, so the result's information content survives only if
4675
+ * its use is adjacent; an entry left empty is replaced with a
4676
+ * placeholder text block.
4677
+ *
4678
+ * Runs as a structural pass over the rendered context in BOTH render
4679
+ * paths — `selectHierarchical` (downstream of `selectL1Summaries`, so
4680
+ * subclass overrides like KnowledgeStrategy are covered) and
4681
+ * `selectAdaptive` (the path FKM defaults onto). It's a no-op on
4682
+ * already-valid output.
4683
+ */
4684
+ enforceToolPairing(entries) {
4685
+ let prevUseIds = new Set();
4686
+ for (let i = 0; i < entries.length; i++) {
4687
+ const entry = entries[i];
4688
+ // --- Rule A: every tool_result must answer a tool_use in the
4689
+ // immediately-preceding entry (and only once). Drop orphans/dupes. ---
4690
+ if (entry.content.some(b => b.type === 'tool_result')) {
4691
+ const seen = new Set();
4692
+ const filtered = entry.content.filter(b => {
4693
+ if (b.type !== 'tool_result')
4694
+ return true;
4695
+ if (!prevUseIds.has(b.toolUseId) || seen.has(b.toolUseId))
4696
+ return false;
4697
+ seen.add(b.toolUseId);
4698
+ return true;
4699
+ });
4700
+ if (filtered.length !== entry.content.length) {
4701
+ entry.content = filtered.length > 0
4702
+ ? filtered
4703
+ : [{ type: 'text', text: '[tool call omitted]' }];
4704
+ }
4705
+ }
4706
+ // --- Rule B: every tool_use in the PREVIOUS entry must be answered
4707
+ // by this entry. Stub any that aren't. ---
4708
+ if (prevUseIds.size > 0) {
4709
+ const answered = new Set();
4710
+ for (const b of entry.content) {
4711
+ if (b.type === 'tool_result')
4712
+ answered.add(b.toolUseId);
4713
+ }
4714
+ const missing = [...prevUseIds].filter(id => !answered.has(id));
4715
+ if (missing.length > 0) {
4716
+ // Look-ahead relocation: the genuine result for a "missing" id is
4717
+ // often sitting a few entries down, displaced by an interleaved
4718
+ // recall pair / pin (it will otherwise be dropped as an orphan by
4719
+ // Rule A when we reach it). Move the real block up into position and
4720
+ // only stub the ids for which no real result exists — so tool output
4721
+ // is preserved, not silently replaced by a placeholder.
4722
+ const results = this.relocateOrDropMissingResults(entries, i, missing);
4723
+ if (answered.size > 0) {
4724
+ // This entry already carries results for the cycle — prepend the
4725
+ // relocated/stub results so all results for the preceding tool_use
4726
+ // sit together (the API wants tool_result blocks at the head of
4727
+ // the message).
4728
+ entry.content = [...results, ...entry.content];
4729
+ }
4730
+ else {
4731
+ // Not a results entry at all — insert a synthetic user entry
4732
+ // between the tool_use entry and this one.
4733
+ entries.splice(i, 0, {
4734
+ index: i,
4735
+ participant: 'user',
4736
+ sourceRelation: 'derived',
4737
+ content: results,
4738
+ });
4739
+ // The stub entry (no tool_use blocks) is now at index i; the
4740
+ // current entry moved to i+1 and is re-processed next iteration
4741
+ // with an empty prevUseIds (its orphan results, if any, were
4742
+ // already filtered above and none survived — Rule A matched
4743
+ // against the same prevUseIds and answered.size === 0).
4744
+ prevUseIds = new Set();
4745
+ continue;
4746
+ }
4747
+ }
4748
+ }
4749
+ prevUseIds = new Set();
4750
+ for (const b of entry.content) {
4751
+ if (b.type === 'tool_use')
4752
+ prevUseIds.add(b.id);
4753
+ }
4754
+ }
4755
+ // Tail: an entry that mixes tool_use with other blocks can survive
4756
+ // trimOrphanedToolUse (which only pops pure use-without-result tails).
4757
+ if (prevUseIds.size > 0) {
4758
+ entries.push({
4759
+ index: entries.length,
4760
+ participant: 'user',
4761
+ sourceRelation: 'derived',
4762
+ content: [...prevUseIds].map(id => ({
4763
+ type: 'tool_result',
4764
+ toolUseId: id,
4765
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
4766
+ })),
4767
+ });
4768
+ }
4769
+ // Reindex after any splices/appends.
4770
+ for (let i = 0; i < entries.length; i++)
4771
+ entries[i].index = i;
4772
+ }
4773
+ /**
4774
+ * For each `missing` tool_use id (a use in the preceding entry with no
4775
+ * adjacent result), return the result block to place next to it:
4776
+ *
4777
+ * - if the genuine result exists within a short look-ahead window after
4778
+ * `afterIndex`, MOVE it up into position (removing it from its source
4779
+ * entry — replacing a now-empty entry with a placeholder) so the real
4780
+ * tool output survives the repair;
4781
+ * - otherwise emit a stub.
4782
+ *
4783
+ * tool_use ids are unique, so the single result carrying a given id is
4784
+ * unambiguously the answer to that use — relocating it cannot break any
4785
+ * other pairing. Results for `missing` ids never sit at or before
4786
+ * `afterIndex` (that entry's results are already matched), so the search
4787
+ * starts at `afterIndex + 1`.
4788
+ */
4789
+ relocateOrDropMissingResults(entries, afterIndex, missing) {
4790
+ const LOOKAHEAD = 6;
4791
+ const end = Math.min(entries.length, afterIndex + 1 + LOOKAHEAD);
4792
+ return missing.map(id => {
4793
+ for (let j = afterIndex + 1; j < end; j++) {
4794
+ const src = entries[j];
4795
+ const bi = src.content.findIndex(b => b.type === 'tool_result' && b.toolUseId === id);
4796
+ if (bi === -1)
4797
+ continue;
4798
+ const real = src.content[bi];
4799
+ const rest = src.content.filter((_, k) => k !== bi);
4800
+ src.content = rest.length > 0
4801
+ ? rest
4802
+ : [{ type: 'text', text: '[tool call omitted]' }];
4803
+ return real;
4804
+ }
4805
+ return {
4806
+ type: 'tool_result',
4807
+ toolUseId: id,
4808
+ content: AutobiographicalStrategy.STUB_TOOL_RESULT_TEXT,
4809
+ };
4810
+ });
4811
+ }
3467
4812
  /**
3468
4813
  * Prune tool_use / tool_result blocks in-place:
3469
4814
  * 1. Truncate `tool_use.input` blocks whose serialized JSON exceeds