@animalabs/context-manager 0.2.0 → 0.4.0

package/dist/src/strategies/autobiographical.js

@@ -39,6 +39,18 @@ export class AutobiographicalStrategy {
     headWindowStartId = null;
     /** Cached result of getHeadWindowStartIndex to avoid repeated linear scans. */
     _cachedHeadStartIndex = null;
+    /** Chronicle store for persistent state. Set in `initialize()`. */
+    store = null;
+    /** Namespace for state-id scoping. Set in `initialize()`. */
+    ns = '';
+    get summariesStateId() { return `${this.ns}/autobio:summaries`; }
+    get counterStateId() { return `${this.ns}/autobio:counter`; }
+    get mergeQueueStateId() { return `${this.ns}/autobio:mergeQueue`; }
+    get pinsStateId() { return `${this.ns}/autobio:pins`; }
+    /** Protected ranges (pins + documents). Loaded from chronicle in initialize. */
+    pins = [];
+    /** Monotonically increasing counter for pin ids. Persisted as part of the pins snapshot. */
+    pinIdCounter = 0;
     constructor(config = {}) {
         this.config = { ...DEFAULT_AUTOBIOGRAPHICAL_CONFIG, ...config };
         // Hierarchical is on by default; set hierarchical: false to use legacy single-level
@@ -52,8 +64,14 @@ export class AutobiographicalStrategy {
         }
     }
     async initialize(ctx) {
+        // Bind to the chronicle store + namespace for persistent strategy state.
+        this.store = ctx.store;
+        this.ns = ctx.namespace;
+        this.registerStates();
+        this.loadPersistedState();
         // Restore headWindowStartId from last topic transition message
         const messages = ctx.messageStore.getAll();
+        this.headWindowStartId = null;
         for (let i = messages.length - 1; i >= 0; i--) {
             if (this.isTopicTransitionMessage(messages[i])) {
                 this.headWindowStartId = messages[i].id;
@@ -62,6 +80,282 @@ export class AutobiographicalStrategy {
         }
         this.rebuildChunks(ctx.messageStore);
     }
+    /**
+     * Register the three Chronicle state slots this strategy uses.
+     * Idempotent — chronicle throws if a state is already registered, which we
+     * swallow (the existing slot is what we want).
+     */
+    registerStates() {
+        if (!this.store)
+            return;
+        try {
+            this.store.registerState({
+                id: this.summariesStateId,
+                strategy: 'append_log',
+                deltaSnapshotEvery: 50,
+                fullSnapshotEvery: 10,
+            });
+        }
+        catch { /* already registered */ }
+        try {
+            this.store.registerState({
+                id: this.counterStateId,
+                strategy: 'snapshot',
+            });
+        }
+        catch { /* already registered */ }
+        try {
+            this.store.registerState({
+                id: this.mergeQueueStateId,
+                strategy: 'snapshot',
+            });
+        }
+        catch { /* already registered */ }
+        try {
+            this.store.registerState({
+                id: this.pinsStateId,
+                strategy: 'snapshot',
+            });
+        }
+        catch { /* already registered */ }
+    }
+    /**
+     * Load summaries, counter, and pending merges from chronicle into the
+     * in-memory mirrors. Called on every (re)initialize so branch switches
+     * pick up the new branch's state.
+     */
+    loadPersistedState() {
+        if (!this.store) {
+            this.summaries = [];
+            this.summaryIdCounter = 0;
+            this.mergeQueue = [];
+            this.pins = [];
+            this.pinIdCounter = 0;
+            return;
+        }
+        const summaries = this.store.getStateJson(this.summariesStateId);
+        this.summaries = Array.isArray(summaries) ? summaries : [];
+        const counter = this.store.getStateJson(this.counterStateId);
+        this.summaryIdCounter = typeof counter === 'number' ? counter : 0;
+        const queue = this.store.getStateJson(this.mergeQueueStateId);
+        this.mergeQueue = Array.isArray(queue)
+            ? queue
+            : [];
+        const pinsState = this.store.getStateJson(this.pinsStateId);
+        if (pinsState && typeof pinsState === 'object' && Array.isArray(pinsState.pins)) {
+            const ps = pinsState;
+            this.pins = ps.pins;
+            this.pinIdCounter = typeof ps.counter === 'number' ? ps.counter : ps.pins.length;
+        }
+        else {
+            this.pins = [];
+            this.pinIdCounter = 0;
+        }
+    }
+    /** Persist the current pins + counter as a single snapshot. */
+    persistPins() {
+        this.store?.setStateJson(this.pinsStateId, {
+            pins: this.pins,
+            counter: this.pinIdCounter,
+        });
+    }
+    // ============================================================================
+    // Pins / documents (protected ranges)
+    // ============================================================================
+    /**
+     * Pin a range of messages so they aren't compressed and render raw at
+     * their original position. Returns the pin id.
+     */
+    pinRange(firstMessageId, lastMessageId, opts) {
+        const id = `pin-${this.pinIdCounter++}`;
+        this.pins.push({
+            id,
+            firstMessageId,
+            lastMessageId,
+            kind: 'pin',
+            name: opts?.name,
+            created: Date.now(),
+        });
+        this.persistPins();
+        return id;
+    }
+    /**
+     * Mark a single message as a "document" — semantically a body of
+     * information the agent wants to retain in full. Functionally a
+     * single-message pin with `kind: 'document'`.
+     */
+    markDocument(messageId, opts) {
+        const id = `pin-${this.pinIdCounter++}`;
+        this.pins.push({
+            id,
+            firstMessageId: messageId,
+            lastMessageId: messageId,
+            kind: 'document',
+            name: opts?.name,
+            created: Date.now(),
+        });
+        this.persistPins();
+        return id;
+    }
+    /** Remove a pin or document mark by id. Returns true if removed. */
+    unpin(pinId) {
+        const before = this.pins.length;
+        this.pins = this.pins.filter(p => p.id !== pinId);
+        if (this.pins.length < before) {
+            this.persistPins();
+            return true;
+        }
+        return false;
+    }
+    /** Read-only list of all current pins. */
+    listPins() {
+        return this.pins;
+    }
+    // ============================================================================
+    // Search (gap #7)
+    // ============================================================================
+    /**
+     * Look up a single summary by id. Returns null if not found.
+     */
+    getSummary(id) {
+        return this.summaries.find(s => s.id === id) ?? null;
+    }
+    /**
+     * Search summaries by substring or regex over their content.
+     *
+     * Result ordering: matches by descending hit count, then by descending
+     * `created` timestamp (newest first within the same hit count).
+     *
+     * Default behavior: only "live" (unmerged) summaries are searched. Set
+     * `includeMerged: true` to also include summaries that have been folded
+     * into a higher level.
+     */
+    searchSummaries(query) {
+        const limit = query.limit ?? 50;
+        const includeMerged = query.includeMerged ?? false;
+        // Build the matcher
+        let matcher = null;
+        if (query.regex) {
+            const flags = query.regex.flags.includes('g') ? query.regex.flags : query.regex.flags + 'g';
+            const re = new RegExp(query.regex.source, flags);
+            matcher = (content) => {
+                const matches = content.match(re);
+                return matches ? matches.length : 0;
+            };
+        }
+        else if (query.text) {
+            const needle = query.text.toLowerCase();
+            matcher = (content) => {
+                const hay = content.toLowerCase();
+                let count = 0;
+                let idx = 0;
+                while ((idx = hay.indexOf(needle, idx)) !== -1) {
+                    count++;
+                    idx += needle.length || 1;
+                }
+                return count;
+            };
+        }
+        else {
+            // No pattern: every summary "matches" once
+            matcher = () => 1;
+        }
+        const levelsFilter = query.levels && query.levels.length > 0 ? new Set(query.levels) : null;
+        const results = [];
+        for (const s of this.summaries) {
+            if (!includeMerged && s.mergedInto)
+                continue;
+            if (levelsFilter && !levelsFilter.has(s.level))
+                continue;
+            const matches = matcher(s.content);
+            if (matches > 0) {
+                results.push({ summary: s, matches });
+            }
+        }
+        results.sort((a, b) => {
+            if (b.matches !== a.matches)
+                return b.matches - a.matches;
+            return b.summary.created - a.summary.created;
+        });
+        return results.slice(0, limit);
+    }
+    /**
+     * Whether a given message position is inside any protected range.
+     * Uses a position map (computed by caller) so callers can avoid
+     * repeated per-message lookups in tight loops.
+     */
+    isPositionPinned(position, pinPositions) {
+        return pinPositions.has(position);
+    }
+    /**
+     * Build a set of message-store positions covered by any pin. O(N pins · K range).
+     * Returns positions for which the message exists; orphan pins (deleted
+     * messages) are silently skipped.
+     */
+    pinnedPositions(messages) {
+        if (this.pins.length === 0)
+            return new Set();
+        const positionOf = new Map();
+        for (let i = 0; i < messages.length; i++) {
+            positionOf.set(messages[i].id, i);
+        }
+        const out = new Set();
+        for (const pin of this.pins) {
+            const first = positionOf.get(pin.firstMessageId);
+            const last = positionOf.get(pin.lastMessageId);
+            if (first === undefined || last === undefined)
+                continue;
+            const lo = Math.min(first, last);
+            const hi = Math.max(first, last);
+            for (let i = lo; i <= hi; i++)
+                out.add(i);
+        }
+        return out;
+    }
+    /**
+     * Append a summary to the in-memory list and to the chronicle AppendLog.
+     * Single point so subclasses inherit persistence.
+     */
+    pushSummary(entry) {
+        this.summaries.push(entry);
+        this.store?.appendToStateJson(this.summariesStateId, entry);
+    }
+    /**
+     * Mark a summary as merged into a higher-level summary, updating the
+     * chronicle copy at the same index. Index is the position in `this.summaries`.
+     */
+    setMergedInto(entry, mergedIntoId) {
+        entry.mergedInto = mergedIntoId;
+        if (!this.store)
+            return;
+        const index = this.summaries.indexOf(entry);
+        if (index < 0)
+            return;
+        this.store.editStateItem(this.summariesStateId, index, Buffer.from(JSON.stringify(entry)));
+    }
+    /**
+     * Allocate the next summary-id counter value and persist the new counter.
+     */
+    nextSummaryIdCounter() {
+        const value = this.summaryIdCounter++;
+        this.store?.setStateJson(this.counterStateId, this.summaryIdCounter);
+        return value;
+    }
+    /**
+     * Push to the merge queue and persist the new queue snapshot.
+     */
+    enqueueMerge(merge) {
+        this.mergeQueue.push(merge);
+        this.store?.setStateJson(this.mergeQueueStateId, this.mergeQueue);
+    }
+    /**
+     * Pop from the merge queue and persist the new queue snapshot.
+     */
+    dequeueMerge() {
+        const merge = this.mergeQueue.shift();
+        this.store?.setStateJson(this.mergeQueueStateId, this.mergeQueue);
+        return merge;
+    }
     checkReadiness() {
         if (this.pendingCompression) {
             return {
@@ -87,13 +381,75 @@ export class AutobiographicalStrategy {
     }
     async onNewMessage(message, ctx) {
         this.rebuildChunks(ctx.messageStore);
-        // Auto-tick: fire compression in the background so it runs without
-        // the framework explicitly calling tick(). compile() will await
-        // pendingCompression via checkReadiness().
-        if (this.config.autoTickOnNewMessage && this.compressionQueue.length > 0 && !this.pendingCompression) {
-            this.tick(ctx).catch((err) => console.error('AutobiographicalStrategy: auto-tick error:', err));
+        // Auto-tick: fire speculative compression in the background. After
+        // each tick completes, if the queue still has work AND we're under
+        // the speculation cap AND preflight allows, schedule another tick.
+        // This drains the queue ahead of need rather than one-chunk-per-
+        // user-turn (reactive). Combined with ContextManager.compile not
+        // awaiting pendingCompression, the agent's response and background
+        // compression run truly in parallel.
+        if (this.config.autoTickOnNewMessage && !this.pendingCompression) {
+            this.driveSpeculativeDrain(ctx);
         }
     }
+    /**
+     * Background-drain loop: keeps calling tick() while there's queued work,
+     * subject to the speculation cap and preflight hook. Recurses via
+     * `queueMicrotask` so one chunk's compression doesn't block the
+     * scheduling of the next.
+     *
+     * Stops if a tick fails to make progress (queue size unchanged) — guards
+     * against runaway recursion when tick is a no-op (e.g. no membrane
+     * configured, or a subclass override that doesn't process the queue).
+     */
+    driveSpeculativeDrain(ctx) {
+        if (this.pendingCompression)
+            return;
+        if (this.compressionQueue.length === 0 && this.mergeQueue.length === 0)
+            return;
+        if (this.isAtSpeculativeCap())
+            return;
+        if (!this.shouldCompressPreflight())
+            return;
+        const beforeChunks = this.compressionQueue.length;
+        const beforeMerges = this.mergeQueue.length;
+        this.tick(ctx)
+            .then(() => {
+            const afterChunks = this.compressionQueue.length;
+            const afterMerges = this.mergeQueue.length;
+            // Made progress if either queue shrank.
+            const progressed = afterChunks < beforeChunks || afterMerges < beforeMerges;
+            if (!progressed)
+                return;
+            // Recurse to drain more. queueMicrotask defers until the current
+            // task is done, letting other code (the agent's stream consumer)
+            // interleave.
+            queueMicrotask(() => this.driveSpeculativeDrain(ctx));
+        })
+            .catch((err) => {
+            console.error('AutobiographicalStrategy: speculative-drain error:', err);
+        });
+    }
+    /**
+     * Whether the strategy's pending+queued L1 budget has reached the cap
+     * configured by `maxSpeculativeL1s`. If no cap is set, always false.
+     */
+    isAtSpeculativeCap() {
+        const cap = this.config.maxSpeculativeL1s;
+        if (cap === undefined || cap < 0)
+            return false;
+        const unmergedL1s = this.summaries.filter(s => s.level === 1 && !s.mergedInto).length;
+        return unmergedL1s + this.compressionQueue.length > cap;
+    }
+    /**
+     * Preflight hook for whether speculative compression should fire on
+     * `onNewMessage`. Returns true by default (current eager behavior).
+     * Subclasses can override for predictive scheduling — e.g. only fire
+     * when the live tail token count is approaching some threshold.
+     */
+    shouldCompressPreflight() {
+        return true;
+    }
     async tick(ctx) {
         if (this.pendingCompression)
             return;
@@ -119,11 +475,26 @@ export class AutobiographicalStrategy {
             return;
         }
         // Priority 2: Execute pending merges (hierarchical only)
+        //
+        // Peek at the head rather than dequeueing eagerly: dequeueMerge persists
+        // the shorter queue *before* the LLM call leaves the building, so a
+        // transient failure (429, network drop, timeout, executeMerge throw)
+        // would silently lose the merge from disk and the sources would sit at
+        // level N-1 with no mergedInto pointers forever. Commit the removal
+        // only after the merge succeeds; on failure, the queue keeps its entry
+        // and the next tick() retries it.
         if (this.config.hierarchical && this.mergeQueue.length > 0) {
-            const merge = this.mergeQueue.shift();
+            const merge = this.mergeQueue[0];
             this.pendingCompression = this.executeMerge(merge.level, merge.sourceIds, ctx);
             try {
                 await this.pendingCompression;
+                // Success: drop from head and persist the shorter queue. We
+                // re-check that head is still our merge in case some future code
+                // path mutates the queue mid-await (today no other site does,
+                // but the assertion makes that invariant explicit).
+                if (this.mergeQueue[0] === merge) {
+                    this.dequeueMerge();
+                }
             }
             finally {
                 this.pendingCompression = null;
@@ -150,6 +521,39 @@ export class AutobiographicalStrategy {
             pendingMerges: this.mergeQueue.length,
         };
     }
+    /**
+     * Richer per-render stats: requires a message store view to compute the
+     * head + tail (recent window) sizes. Returns counts AND token sums per
+     * summary level, so observers can see "how much of the agent's context
+     * is in raw tail vs folded into L1/L2/L3."
+     *
+     * Useful for TUI / dashboards. The token sums use the strategy's own
+     * token estimates (which match what `select()` uses for budget math).
+     */
+    getRenderStats(store) {
+        const messages = store.getAll();
+        const headStart = this.getHeadWindowStartIndex(store);
+        const headEnd = this.getHeadWindowEnd(store);
+        const recentStart = this.getRecentWindowStart(store);
+        const sumTokens = (slice) => slice.reduce((acc, m) => acc + store.estimateTokens(m), 0);
+        const headMsgs = messages.slice(headStart, headEnd);
+        const tailMsgs = messages.slice(recentStart);
+        const live = (level) => this.summaries.filter(s => s.level === level && !s.mergedInto);
+        const sumLevelTokens = (level) => live(level).reduce((acc, s) => acc + s.tokens, 0);
+        return {
+            head: { messages: headMsgs.length, tokens: sumTokens(headMsgs) },
+            tail: { messages: tailMsgs.length, tokens: sumTokens(tailMsgs) },
+            summaries: {
+                l1: { count: live(1).length, tokens: sumLevelTokens(1) },
+                l2: { count: live(2).length, tokens: sumLevelTokens(2) },
+                l3: { count: live(3).length, tokens: sumLevelTokens(3) },
+            },
+            pending: {
+                chunks: this.chunks.filter(c => !c.compressed).length,
+                merges: this.mergeQueue.length,
+            },
+        };
+    }
     // ============================================================================
     // Legacy (single-level) path
     // ============================================================================
@@ -166,7 +570,7 @@ export class AutobiographicalStrategy {
             const msg = messages[i];
             const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
             const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-            if (totalTokens + tokens > maxTokens)
+            if (this.isOverBudget(totalTokens + tokens, maxTokens))
                 break;
             entries.push({
                 index: entries.length,
@@ -197,15 +601,18 @@ export class AutobiographicalStrategy {
                     content: [{ type: 'text', text: contextLabel }],
                     sourceRelation: 'derived',
                 };
+                // Synthesised summary turns must respect maxMessageTokens just like raw
+                // copies do — otherwise a runaway diary can starve recent messages.
+                const answerContent = [{ type: 'text', text: chunk.diary }];
                 const answerEntry = {
                     index: entries.length + 1,
                     participant: summaryParticipant,
-                    content: [{ type: 'text', text: chunk.diary }],
+                    content: msgCap > 0 ? this.truncateContent(answerContent, msgCap) : answerContent,
                     sourceRelation: 'derived',
                 };
                 const pairTokens = this.estimateTokens(questionEntry.content) +
                     this.estimateTokens(answerEntry.content);
-                if (totalTokens + pairTokens > maxTokens)
+                if (this.isOverBudget(totalTokens + pairTokens, maxTokens))
                     break;
                 entries.push(questionEntry);
                 entries.push(answerEntry);
@@ -216,7 +623,7 @@ export class AutobiographicalStrategy {
                 for (const msg of chunk.messages) {
                     const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
                     const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-                    if (totalTokens + tokens > maxTokens)
+                    if (this.isOverBudget(totalTokens + tokens, maxTokens))
                         break;
                     entries.push({
                         index: entries.length,
@@ -240,7 +647,7 @@ export class AutobiographicalStrategy {
             const msg = messages[i];
             const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
             const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-            if (totalTokens + tokens > maxTokens)
+            if (this.isOverBudget(totalTokens + tokens, maxTokens))
                 break;
             entries.push({
                 index: entries.length,
@@ -253,12 +660,47 @@ export class AutobiographicalStrategy {
         }
         // 3. Recent uncompressed messages (skip those already in head window)
         const recentStart = Math.max(this.getRecentWindowStart(store), headEnd);
-        for (let i = recentStart; i < messages.length; i++) {
+        this.emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokens);
+        this.trimOrphanedToolUse(entries);
+        this.pruneToolEntries(entries);
+        return entries;
+    }
+    /**
+     * Emit recent-window messages, evicting OLDEST-first when the budget is tight.
+     *
+     * The previous loop iterated `recentStart → messages.length` forward and broke
+     * on `totalTokens + tokens > maxTokens`. When the head/summary section eats most
+     * of the budget, the loop emits the oldest messages of the window and aborts
+     * before reaching the newest — exactly the messages an agent needs to act on.
+     * This helper picks newest-first within the budget, then emits the kept set in
+     * chronological order, dropping a leading orphan tool_result if its tool_use
+     * fell into the evicted older portion.
+     */
+    emitRecentNewestFirst(entries, store, messages, recentStart, msgCap, maxTokens, totalTokensBefore) {
+        if (recentStart >= messages.length)
+            return;
+        const accepted = [];
+        let acceptedTokens = 0;
+        for (let i = messages.length - 1; i >= recentStart; i--) {
             const msg = messages[i];
-            const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
-            const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-            if (totalTokens + tokens > maxTokens)
+            const tokens = msgCap > 0
+                ? Math.min(store.estimateTokens(msg), msgCap + 50)
+                : store.estimateTokens(msg);
+            if (this.isOverBudget(totalTokensBefore + acceptedTokens + tokens, maxTokens))
                 break;
+            accepted.push(i);
+            acceptedTokens += tokens;
+        }
+        accepted.reverse();
+        // Drop leading orphan tool_result(s): their matching tool_use was evicted.
+        while (accepted.length > 0 &&
+            this.hasToolResult(messages[accepted[0]]) &&
+            !this.hasToolUse(messages[accepted[0]])) {
+            accepted.shift();
+        }
+        for (const i of accepted) {
+            const msg = messages[i];
+            const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
             entries.push({
                 index: entries.length,
                 sourceMessageId: msg.id,
@@ -266,10 +708,7 @@ export class AutobiographicalStrategy {
                 participant: msg.participant,
                 content,
             });
-            totalTokens += tokens;
         }
-        this.trimOrphanedToolUse(entries);
-        return entries;
     }
     async compressChunkLegacy(chunk, ctx) {
         if (!ctx.membrane) {
@@ -295,7 +734,6 @@ export class AutobiographicalStrategy {
             config: {
                 model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
                 maxTokens: 2000,
-                temperature: 0,
             },
         };
         try {
@@ -416,7 +854,6 @@ export class AutobiographicalStrategy {
             config: {
                 model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
                 maxTokens: Math.round(targetTokens * 1.5),
-                temperature: 0,
             },
         };
         try {
@@ -427,7 +864,7 @@ export class AutobiographicalStrategy {
                 .join('\n');
             const messageIds = chunk.messages.map(m => m.id);
             const entry = {
-                id: `L1-${this.summaryIdCounter++}`,
+                id: `L1-${this.nextSummaryIdCounter()}`,
                 level: 1,
                 content: summaryText,
                 tokens: Math.ceil(summaryText.length / 4),
@@ -440,7 +877,7 @@ export class AutobiographicalStrategy {
                 created: Date.now(),
                 phaseType: chunk.phaseType,
             };
-            this.summaries.push(entry);
+            this.pushSummary(entry);
             chunk.compressed = true;
             chunk.summaryId = entry.id;
             this._compressionCount++;
@@ -454,23 +891,37 @@ export class AutobiographicalStrategy {
     /**
      * Check if unmerged summary counts exceed the merge threshold.
      * Enqueues merge operations if so.
+     *
+     * Skips L1s/L2s that are already in a pending merge — without this guard,
+     * each new summary above threshold re-enqueues a merge for the same
+     * already-eligible siblings, producing N near-identical higher-level
+     * summaries when the queue eventually drains.
      */
     checkMergeThreshold() {
         const threshold = this.config.mergeThreshold ?? 6;
+        // IDs that are already part of a queued merge — exclude them from
+        // eligibility so we don't re-enqueue.
+        const queuedL1 = new Set();
+        const queuedL2 = new Set();
+        for (const m of this.mergeQueue) {
+            const set = m.level === 2 ? queuedL1 : queuedL2;
+            for (const id of m.sourceIds)
+                set.add(id);
+        }
         // Check L1 → L2
-        const unmergedL1 = this.summaries.filter(s => s.level === 1 && !s.mergedInto);
+        const unmergedL1 = this.summaries.filter(s => s.level === 1 && !s.mergedInto && !queuedL1.has(s.id));
         if (unmergedL1.length >= threshold) {
             const toMerge = unmergedL1.slice(0, threshold);
-            this.mergeQueue.push({
+            this.enqueueMerge({
                 level: 2,
                 sourceIds: toMerge.map(s => s.id),
             });
         }
         // Check L2 → L3
-        const unmergedL2 = this.summaries.filter(s => s.level === 2 && !s.mergedInto);
+        const unmergedL2 = this.summaries.filter(s => s.level === 2 && !s.mergedInto && !queuedL2.has(s.id));
         if (unmergedL2.length >= threshold) {
             const toMerge = unmergedL2.slice(0, threshold);
-            this.mergeQueue.push({
+            this.enqueueMerge({
                 level: 3,
                 sourceIds: toMerge.map(s => s.id),
             });
@@ -491,6 +942,14 @@ export class AutobiographicalStrategy {
             console.warn('executeMerge: some source summaries not found, skipping');
             return;
         }
+        // Defensive: if every source is already mergedInto something, this is a
+        // stale queue entry (could happen if multiple merges for the same
+        // sourceIds were enqueued before the dedup fix in checkMergeThreshold).
+        // Skip rather than produce a redundant near-identical higher-level entry.
+        if (sources.every(s => s.mergedInto)) {
+            console.warn(`executeMerge: all sources already merged into ${sources[0].mergedInto}, skipping (stale queue entry)`);
+            return;
+        }
         const targetTokens = this.config.summaryTargetTokens ?? 2000;
         const participant = this.config.summaryParticipant ?? 'Claude';
         // Build message array
@@ -530,7 +989,6 @@ export class AutobiographicalStrategy {
             config: {
                 model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
                 maxTokens: Math.round(targetTokens * 1.5),
-                temperature: 0,
             },
         };
         try {
@@ -546,7 +1004,7 @@ export class AutobiographicalStrategy {
             };
             const sourceLevel = (targetLevel - 1);
             const newEntry = {
-                id: `L${targetLevel}-${this.summaryIdCounter++}`,
+                id: `L${targetLevel}-${this.nextSummaryIdCounter()}`,
                 level: targetLevel,
                 content: mergedText,
                 tokens: Math.ceil(mergedText.length / 4),
@@ -555,10 +1013,14 @@ export class AutobiographicalStrategy {
                 sourceRange,
                 created: Date.now(),
             };
-            this.summaries.push(newEntry);
-            // Mark sources as merged
+            // Append the new merged entry first, then mark sources. Persist each
+            // mergedInto edit individually so chronicle reflects the same shape as
+            // the in-memory mirror. (If the process crashes mid-loop, restart sees
+            // the new entry plus a partial set of marked sources; un-marked sources
+            // would re-trigger a merge — accept the rare duplicate over data loss.)
+            this.pushSummary(newEntry);
             for (const source of sources) {
-                source.mergedInto = newEntry.id;
+                this.setMergedInto(source, newEntry.id);
             }
             // Check if this merge triggers a further merge
             this.checkMergeThreshold();
@@ -585,7 +1047,7 @@ export class AutobiographicalStrategy {
             const msg = messages[i];
             const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
             const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-            if (totalTokens + tokens > maxTokens)
+            if (this.isOverBudget(totalTokens + tokens, maxTokens))
                 break;
             entries.push({
                 index: entries.length,
@@ -620,7 +1082,7 @@ export class AutobiographicalStrategy {
         for (const s of shownL3) {
             if (l3Used + s.tokens > l3Budget)
                 break;
-            if (totalTokens + totalSummaryTokens + s.tokens > maxTokens)
+            if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
                 break;
             selectedSummaries.push(s);
             l3Used += s.tokens;
@@ -633,7 +1095,7 @@ export class AutobiographicalStrategy {
         for (const s of shownL2) {
             if (l2Used + s.tokens > l2Effective)
                 break;
-            if (totalTokens + totalSummaryTokens + s.tokens > maxTokens)
+            if (this.isOverBudget(totalTokens + totalSummaryTokens + s.tokens, maxTokens))
                 break;
             selectedSummaries.push(s);
             l2Used += s.tokens;
@@ -646,46 +1108,182 @@ export class AutobiographicalStrategy {
         const { selected: l1Selected, tokensUsed: l1Used } = this.selectL1Summaries(shownL1, l1Effective, l1Remaining);
         selectedSummaries.push(...l1Selected);
         totalSummaryTokens += l1Used;
-        // Emit summaries as a single Q&A pair
-        if (selectedSummaries.length > 0) {
-            const contextLabel = this.config.summaryContextLabel ?? 'What do you remember from earlier?';
-            const combinedText = selectedSummaries.map(s => s.content).join('\n\n---\n\n');
-            const questionEntry = {
-                index: entries.length,
-                participant: 'Context Manager',
-                content: [{ type: 'text', text: contextLabel }],
-                sourceRelation: 'derived',
-            };
-            const answerEntry = {
-                index: entries.length + 1,
-                participant: this.config.summaryParticipant ?? 'Claude',
-                content: [{ type: 'text', text: combinedText }],
-                sourceRelation: 'derived',
-            };
-            const pairTokens = this.estimateTokens(questionEntry.content) +
-                this.estimateTokens(answerEntry.content);
-            entries.push(questionEntry);
-            entries.push(answerEntry);
-            totalTokens += pairTokens;
+        // Emit summaries + pinned messages between head and recent windows.
+        //
+        // Default (positionedRecallPairs=true): one Q/A pair per summary,
+        // interleaved with raw pinned messages, all sorted chronologically by
+        // source-range / message position. Each memory appears in its temporal
+        // place rather than as a wall of unrelated recollections.
+        //
+        // Legacy (positionedRecallPairs=false): summaries concatenated into one
+        // Q/A pair between head and tail; pinned messages still emit raw, in
+        // their chronological positions, after the combined recall pair.
+        const positionOf = new Map();
+        for (let i = 0; i < messages.length; i++) {
+            positionOf.set(messages[i].id, i);
         }
-        // Phase 4: Recent uncompressed messages (skip head window overlap)
-        const effectiveRecentStart = Math.max(recentStart, headEnd);
-        for (let i = effectiveRecentStart; i < messages.length; i++) {
-            const msg = messages[i];
-            const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
-            const tokens = msgCap > 0 ? Math.min(store.estimateTokens(msg), msgCap + 50) : store.estimateTokens(msg);
-            if (totalTokens + tokens > maxTokens)
-                break;
-            entries.push({
-                index: entries.length,
-                sourceMessageId: msg.id,
-                sourceRelation: 'copy',
-                participant: msg.participant,
-                content,
-            });
-            totalTokens += tokens;
+        const pinnedPositionsSet = this.pinnedPositions(messages);
+        // Pinned messages between head and recent (head/recent pinned ones
+        // already emit raw via Phase 0 / Phase 4).
+        const pinnedInMiddle = [];
+        const pinnedIdsInMiddle = new Set();
+        for (let i = headEnd; i < recentStart; i++) {
+            if (pinnedPositionsSet.has(i)) {
+                pinnedInMiddle.push({ msg: messages[i], position: i });
+                pinnedIdsInMiddle.add(messages[i].id);
+            }
+        }
+        // Uncompressed-chunk fallback: messages in the middle region whose
+        // chunk hasn't been summarized yet. Without this, a message that
+        // rolled out of the recent window into a queued-but-not-yet-compressed
+        // chunk would vanish from rendered context — there'd be no summary to
+        // emit (compression hasn't run) and Phase 4 only walks recentStart
+        // onwards. Mirrors selectLegacy's "Uncompressed: emit raw" behavior
+        // around line 738, but here we interleave chronologically with
+        // summaries and pins via the unified items list below.
+        //
+        // This matters because compile() was made non-blocking in commit
+        // `3e42e98` (drops the prior `await readiness.pendingWork`); without
+        // this fallback, the trade was silent data-loss for messages caught
+        // in the queued-but-not-yet-compressed window. Now compile()'s
+        // freshness contract is: summaries may lag the very latest L1, but
+        // no message ever disappears.
+        const uncompressedInMiddle = [];
+        for (const chunk of this.chunks) {
+            if (chunk.compressed)
+                continue;
+            for (const msg of chunk.messages) {
+                const pos = positionOf.get(msg.id);
+                if (pos === undefined)
+                    continue;
+                if (pos < headEnd || pos >= recentStart)
+                    continue;
+                if (pinnedIdsInMiddle.has(msg.id))
+                    continue;
+                uncompressedInMiddle.push({ msg, position: pos });
+            }
+        }
+        // Merged list of raw messages to emit in the middle region —
+        // either a pin or a message whose chunk hasn't compressed yet.
+        // Both render identically (raw, at their chronological position).
+        const middleRaw = [
+            ...pinnedInMiddle,
+            ...uncompressedInMiddle,
+        ];
+        if (selectedSummaries.length > 0 || middleRaw.length > 0) {
+            const summaryParticipant = this.config.summaryParticipant ?? 'Claude';
+            if (this.config.positionedRecallPairs !== false) {
+                const items = [];
+                for (const s of selectedSummaries) {
+                    const pos = positionOf.get(s.sourceRange.first) ?? Number.MAX_SAFE_INTEGER;
+                    items.push({ kind: 'summary', position: pos, summary: s });
+                }
+                for (const p of middleRaw) {
+                    items.push({ kind: 'pin', position: p.position, msg: p.msg });
+                }
+                items.sort((a, b) => a.position - b.position);
+                for (const item of items) {
+                    if (item.kind === 'summary') {
+                        const summary = item.summary;
+                        const headerText = this.buildRecallHeader(summary);
+                        const questionEntry = {
+                            index: entries.length,
+                            participant: 'Context Manager',
+                            content: [{ type: 'text', text: headerText }],
+                            sourceRelation: 'derived',
+                        };
+                        const answerContent = [{ type: 'text', text: summary.content }];
+                        const answerEntry = {
+                            index: entries.length + 1,
+                            participant: summaryParticipant,
+                            content: msgCap > 0 ? this.truncateContent(answerContent, msgCap) : answerContent,
+                            sourceRelation: 'derived',
+                        };
+                        const pairTokens = this.estimateTokens(questionEntry.content) +
+                            this.estimateTokens(answerEntry.content);
+                        if (this.isOverBudget(totalTokens + pairTokens, maxTokens))
+                            break;
+                        entries.push(questionEntry);
+                        entries.push(answerEntry);
+                        totalTokens += pairTokens;
+                    }
+                    else {
+                        const msg = item.msg;
+                        const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
+                        const tokens = msgCap > 0
+                            ? Math.min(store.estimateTokens(msg), msgCap + 50)
+                            : store.estimateTokens(msg);
+                        if (this.isOverBudget(totalTokens + tokens, maxTokens))
+                            break;
+                        entries.push({
+                            index: entries.length,
+                            sourceMessageId: msg.id,
+                            sourceRelation: 'copy',
+                            participant: msg.participant,
+                            content,
+                        });
+                        totalTokens += tokens;
+                    }
+                }
+            }
+            else {
+                // Legacy combined-pair mode for summaries; pins still emit raw at
+                // their positions after the combined pair.
+                if (selectedSummaries.length > 0) {
+                    const contextLabel = this.config.summaryContextLabel ?? 'What do you remember from earlier?';
+                    const combinedText = selectedSummaries.map(s => s.content).join('\n\n---\n\n');
+                    const questionEntry = {
+                        index: entries.length,
+                        participant: 'Context Manager',
+                        content: [{ type: 'text', text: contextLabel }],
+                        sourceRelation: 'derived',
+                    };
+                    // Synthesised summary turns must respect maxMessageTokens. With L1+L2+L3
+                    // budgets defaulting to 30k each, an unconstrained concatenation can push
+                    // a single assistant turn past 90k tokens, eating the inference budget
+                    // and starving recent messages (postmortem 2026-05-04, bug B).
+                    const answerContent = [{ type: 'text', text: combinedText }];
+                    const answerEntry = {
+                        index: entries.length + 1,
+                        participant: summaryParticipant,
+                        content: msgCap > 0 ? this.truncateContent(answerContent, msgCap) : answerContent,
+                        sourceRelation: 'derived',
+                    };
+                    const pairTokens = this.estimateTokens(questionEntry.content) +
+                        this.estimateTokens(answerEntry.content);
+                    entries.push(questionEntry);
+                    entries.push(answerEntry);
+                    totalTokens += pairTokens;
+                }
+                // Sort by position so uncompressed-middle messages and pins both
+                // appear in their chronological place after the combined recall pair.
+                const middleRawSorted = [...middleRaw].sort((a, b) => a.position - b.position);
+                for (const { msg } of middleRawSorted) {
+                    const content = msgCap > 0 ? this.truncateContent(msg.content, msgCap) : msg.content;
+                    const tokens = msgCap > 0
+                        ? Math.min(store.estimateTokens(msg), msgCap + 50)
+                        : store.estimateTokens(msg);
+                    if (this.isOverBudget(totalTokens + tokens, maxTokens))
+                        break;
+                    entries.push({
+                        index: entries.length,
+                        sourceMessageId: msg.id,
+                        sourceRelation: 'copy',
+                        participant: msg.participant,
+                        content,
+                    });
+                    totalTokens += tokens;
+                }
+            }
         }
+        // Phase 4: Recent uncompressed messages (skip head window overlap).
+        // Newest-first eviction so that when summaries/head consume most of the
+        // budget, the latest messages (the ones the agent actually needs to act
+        // on) are preserved and the oldest recent-window messages are dropped.
+        const effectiveRecentStart = Math.max(recentStart, headEnd);
+        this.emitRecentNewestFirst(entries, store, messages, effectiveRecentStart, msgCap, maxTokens, totalTokens);
         this.trimOrphanedToolUse(entries);
+        this.pruneToolEntries(entries);
         return entries;
     }
     // ============================================================================
@@ -715,13 +1313,56 @@ export class AutobiographicalStrategy {
         for (const s of shownL1) {
             if (used + s.tokens > budget)
                 break;
-            if (used + s.tokens > maxTokens)
+            if (this.isOverBudget(used + s.tokens, maxTokens))
                 break;
             selected.push(s);
             used += s.tokens;
         }
         return { selected, tokensUsed: used };
     }
+    /**
+     * True if `projectedTotal` exceeds `max` AND the strategy is configured
+     * to enforce budget. When `enforceBudget: false`, always returns false
+     * — the rendering pipeline emits the full ideal context regardless of
+     * budget overage. Caller's API will reject if it exceeds the model's
+     * context window; the philosophy is "surface overage, don't hide it."
+     */
+    isOverBudget(projectedTotal, max) {
+        if (this.config.enforceBudget === false)
+            return false;
+        return projectedTotal > max;
+    }
+    /**
+     * Sort selected summaries by source-range start position, so per-pair
+     * recall emission appears in chronological order. Falls back to the
+     * created timestamp for summaries whose source-range first message is
+     * no longer in the store.
+     */
+    sortSummariesChronologically(summaries, messages) {
+        const positionOf = new Map();
+        for (let i = 0; i < messages.length; i++) {
+            positionOf.set(messages[i].id, i);
+        }
+        return [...summaries].sort((a, b) => {
+            const posA = positionOf.get(a.sourceRange.first) ?? Number.MAX_SAFE_INTEGER;
+            const posB = positionOf.get(b.sourceRange.first) ?? Number.MAX_SAFE_INTEGER;
+            if (posA !== posB)
+                return posA - posB;
+            return a.created - b.created;
+        });
+    }
+    /**
+     * Render the per-pair recall header from the configured template.
+     * Substitutions: {id} {level} {first} {last}.
+     */
+    buildRecallHeader(summary) {
+        const template = this.config.recallHeaderTemplate ?? '[Recall {id}]';
+        return template
+            .replace(/\{id\}/g, summary.id)
+            .replace(/\{level\}/g, String(summary.level))
+            .replace(/\{first\}/g, summary.sourceRange.first)
+            .replace(/\{last\}/g, summary.sourceRange.last);
+    }
     // ============================================================================
     // Head window reset / topic transition
     // ============================================================================
@@ -781,7 +1422,6 @@ export class AutobiographicalStrategy {
             config: {
                 model: this.config.compressionModel ?? 'claude-sonnet-4-20250514',
                 maxTokens: 1500,
-                temperature: 0,
             },
         };
         const response = await ctx.membrane.complete(request, { formatter: this.nativeFormatter });
@@ -810,16 +1450,26 @@ export class AutobiographicalStrategy {
     // Shared utilities
     // ============================================================================
     /**
-     * Get messages in the compressible zone: outside both head window and recent window.
-     * Returns messages from [0, headStart) ∪ [headEnd, recentStart).
+     * Get messages in the compressible zone: outside both head window and
+     * recent window AND not inside any pinned range. Returns messages from
+     * [0, headStart) ∪ [headEnd, recentStart) minus any positions covered
+     * by a pin or document mark.
      */
     getCompressibleMessages(store) {
         const messages = store.getAll();
         const headStart = this.getHeadWindowStartIndex(store);
         const headEnd = this.getHeadWindowEnd(store);
         const recentStart = this.getRecentWindowStart(store);
-        return messages.slice(0, recentStart)
-            .filter((_, i) => i < headStart || i >= headEnd);
+        const pinned = this.pinnedPositions(messages);
+        const out = [];
+        for (let i = 0; i < recentStart; i++) {
+            if (i >= headStart && i < headEnd)
+                continue;
+            if (pinned.has(i))
+                continue;
+            out.push(messages[i]);
+        }
+        return out;
     }
     /**
      * Rebuild chunk boundaries based on current messages.
@@ -984,6 +1634,92 @@ export class AutobiographicalStrategy {
             }
         }
     }
+    /**
+     * Prune tool_use / tool_result blocks in-place:
+     *  1. Truncate `tool_use.input` blocks whose serialized JSON exceeds
+     *     `toolUseInputMaxTokens`.
+     *  2. For each tool name, keep only the last N `tool_result` blocks
+     *     per `toolResultMaxLastN`; older ones get their `content` replaced
+     *     with a brief marker referencing the tool name and how many newer
+     *     results exist below.
+     *
+     * Both passes are no-ops when the corresponding config is unset/0.
+     * Pruning runs AFTER selection and orphan-trimming, so it doesn't
+     * affect chunk formation or the recall/pin layout.
+     */
+    pruneToolEntries(entries) {
+        // Pass 1: build toolUseId → toolName map and apply input truncation
+        const toolUseInputCap = this.config.toolUseInputMaxTokens ?? 0;
+        const toolUseIdToName = new Map();
+        for (const entry of entries) {
+            for (let i = 0; i < entry.content.length; i++) {
+                const block = entry.content[i];
+                if (block.type !== 'tool_use')
+                    continue;
+                toolUseIdToName.set(block.id, block.name);
+                if (toolUseInputCap > 0) {
+                    const inputJson = JSON.stringify(block.input);
+                    const inputTokens = Math.ceil(inputJson.length / 4);
+                    if (inputTokens > toolUseInputCap) {
+                        const keys = Object.keys(block.input).slice(0, 5);
+                        entry.content[i] = {
+                            ...block,
+                            input: {
+                                _truncated: true,
+                                _originalTokens: inputTokens,
+                                _keys: keys,
+                            },
+                        };
+                    }
+                }
+            }
+        }
+        // Pass 2: collect tool_result occurrences per tool name, in order
+        const occurrencesByTool = new Map();
+        for (const entry of entries) {
+            for (let i = 0; i < entry.content.length; i++) {
+                const block = entry.content[i];
+                if (block.type !== 'tool_result')
+                    continue;
+                const toolName = toolUseIdToName.get(block.toolUseId);
+                if (!toolName)
+                    continue;
+                let arr = occurrencesByTool.get(toolName);
+                if (!arr) {
+                    arr = [];
+                    occurrencesByTool.set(toolName, arr);
+                }
+                arr.push({ entry, blockIndex: i });
+            }
+        }
+        // Pass 3: apply per-tool max-last-N
+        const cfg = this.config.toolResultMaxLastN;
+        if (cfg === undefined)
+            return;
+        for (const [toolName, occs] of occurrencesByTool) {
+            let limit;
+            if (typeof cfg === 'number')
+                limit = cfg;
+            else if (typeof cfg === 'object')
+                limit = cfg[toolName];
+            if (limit === undefined || limit < 0)
+                continue;
+            const excessCount = occs.length - limit;
+            if (excessCount <= 0)
+                continue;
+            for (let i = 0; i < excessCount; i++) {
+                const { entry, blockIndex } = occs[i];
+                const orig = entry.content[blockIndex];
+                if (orig.type !== 'tool_result')
+                    continue;
+                const fresherCount = occs.length - i - 1;
+                entry.content[blockIndex] = {
+                    ...orig,
+                    content: `[Result truncated — tool '${toolName}' has ${fresherCount} more recent result${fresherCount === 1 ? '' : 's'} below]`,
+                };
+            }
+        }
+    }
     isChunkOldEnough(chunk) {
         return true;
     }