@qearlyao/familiar 0.2.3 → 0.2.5

package/dist/inbound-attachments.js

@@ -4,10 +4,13 @@ import { basename, extname, resolve } from "node:path";
 import { attachmentsDir, publicAttachmentPath } from "./generated-media.js";
 import { ensureInlineImageDerivative, MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
 import { deriveInboundAttachmentText } from "./media-understanding.js";
+import { IMAGE_EXTENSION_BY_MIME, sniffImageMimeType } from "./util/image-mime.js";
 export { MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
 export const MAX_INBOUND_ATTACHMENTS = 4;
 export const MAX_INBOUND_ATTACHMENT_BYTES = 12 * 1024 * 1024;
 export const MAX_INBOUND_TOTAL_BYTES = 24 * 1024 * 1024;
+const TEXT_ATTACHMENT_PREVIEW_LINES = 2;
+const TEXT_ATTACHMENT_PREVIEW_CHARS = 1000;
 const ALLOWED_MIME_TYPES = new Set([
     "image/jpeg",
     "image/png",
@@ -23,10 +26,7 @@ const ALLOWED_MIME_TYPES = new Set([
     "text/plain",
 ]);
 const EXTENSIONS_BY_MIME = {
-    "image/jpeg": ".jpg",
-    "image/png": ".png",
-    "image/gif": ".gif",
-    "image/webp": ".webp",
+    ...IMAGE_EXTENSION_BY_MIME,
     "audio/mpeg": ".mp3",
     "audio/ogg": ".ogg",
     "audio/wav": ".wav",
@@ -51,32 +51,42 @@ function kindFromMime(mimeType) {
         return "video";
     return "file";
 }
+function textAttachmentPreview(buffer, mimeType) {
+    if (mimeType !== "text/plain")
+        return undefined;
+    const decoded = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
+    const normalized = decoded.replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
+    if (!normalized || normalized.includes("\uFFFD"))
+        return undefined;
+    const lines = normalized.split("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_LINES);
+    const preview = lines.join("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_CHARS).trim();
+    return preview || undefined;
+}
 function sniffText(buffer) {
     if (buffer.length === 0)
         return "text/plain";
     const head = buffer.subarray(0, Math.min(buffer.length, 512));
     if (head.includes(0))
         return undefined;
-    return head.every((byte) => byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126))
-        ? "text/plain"
-        : undefined;
-}
-function sniffMimeType(buffer, declared) {
-    let detected;
-    if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
-        detected = "image/jpeg";
-    else if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
-        detected = "image/png";
+    let decoded;
+    try {
+        decoded = new TextDecoder("utf-8", { fatal: true }).decode(head);
     }
-    else if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" ||
-        buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
-        detected = "image/gif";
+    catch {
+        return undefined;
     }
-    else if (buffer.subarray(0, 4).toString("ascii") === "RIFF" &&
-        buffer.subarray(8, 12).toString("ascii") === "WEBP") {
-        detected = "image/webp";
+    for (const char of decoded) {
+        const code = char.codePointAt(0) ?? 0;
+        if (code === 9 || code === 10 || code === 13)
+            continue;
+        if (code < 32 || code === 127)
+            return undefined;
     }
-    else if (buffer.subarray(0, 4).toString("ascii") === "%PDF")
+    return "text/plain";
+}
+function sniffMimeType(buffer, declared) {
+    let detected = sniffImageMimeType(buffer);
+    if (!detected && buffer.subarray(0, 4).toString("ascii") === "%PDF")
         detected = "application/pdf";
     else if (buffer.subarray(0, 3).toString("ascii") === "ID3" ||
         buffer.subarray(0, 2).equals(Buffer.from([0xff, 0xfb]))) {
@@ -204,6 +214,18 @@ export async function materializeInboundAttachments(config, inputs) {
                 source: attachment.source,
                 sha256: attachment.sha256,
             };
+            const textPreview = textAttachmentPreview(attachment.buffer, attachment.mimeType);
+            if (textPreview) {
+                finalAttachment.derived = {
+                    ...finalAttachment.derived,
+                    text: {
+                        provider: "local",
+                        model: "text-preview",
+                        label: "preview",
+                        text: textPreview,
+                    },
+                };
+            }
             const derivedImage = await ensureInlineImageDerivative(config, finalAttachment);
             if (derivedImage) {
                 if (derivedImage.localPath && !existingDerivedPaths.has(derivedImage.localPath)) {
@@ -260,7 +282,16 @@ export async function promptImagesFromAttachments(attachments) {
 export function promptAttachmentNotes(attachments) {
     return attachments
         .map((attachment) => {
-        const attrs = `name="${attachment.name}" id="${attachment.id}" kind="${attachment.kind ?? "file"}" mime="${attachment.mimeType ?? "unknown"}" size="${attachment.size ?? "unknown"}"`;
+        const attrs = [
+            `name="${attachment.name}"`,
+            `id="${attachment.id}"`,
+            `kind="${attachment.kind ?? "file"}"`,
+            `mime="${attachment.mimeType ?? "unknown"}"`,
+            `size="${attachment.size ?? "unknown"}"`,
+            attachment.localPath ? `path="${attachment.localPath}"` : undefined,
+        ]
+            .filter(Boolean)
+            .join(" ");
         const derivedText = attachment.derived?.text?.text;
         if (derivedText) {
             const label = attachment.derived?.text?.label || (attachment.kind === "audio" ? "transcription" : "summary");

package/dist/memory/diary/ambient-injector.js

@@ -1,3 +1,4 @@
+import { positiveIntegerOrDefault } from "../util.js";
 import { retrieveAmbientDiary } from "./ambient.js";
 const INJECTED_MEMORY_OPEN = "<injected_memory>";
 const INJECTED_MEMORY_CLOSE = "</injected_memory>";
@@ -64,9 +65,6 @@ export class AmbientDiaryInjector {
         }
     }
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}
 function nonNegativeIntegerOrDefault(value, fallback) {
     return value !== undefined && Number.isInteger(value) && value >= 0 ? value : fallback;
 }

package/dist/memory/diary/ambient.js

@@ -1,4 +1,5 @@
 import { retrieveMemory, } from "../index/retrieval.js";
+import { positiveIntegerOrDefault } from "../util.js";
 import { DIARY_CHUNK_CORPUS } from "./chunks.js";
 const DEFAULT_LIMIT = 4;
 const DEFAULT_CANDIDATE_MULTIPLIER = 5;
@@ -119,6 +120,3 @@ function normalizeUnit(value) {
     const absolute = Math.abs(value);
     return Math.max(0, Math.min(1, absolute > 1 ? absolute / 10 : absolute));
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/diary/chunks.js

@@ -1,4 +1,5 @@
 import { basename } from "node:path";
+import { positiveIntegerOrDefault } from "../util.js";
 export const DIARY_CHUNK_CORPUS = "diary_chunk";
 const DEFAULT_MAX_CHARS = 2400;
 const DIARY_DATE_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
@@ -226,6 +227,3 @@ function stripInlineMarkdown(value) {
 function isMarkdownHeading(line) {
     return /^#{1,6}\s+/.test(line);
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/diary/indexer.js

@@ -1,5 +1,6 @@
 import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, isAbsolute, join, resolve } from "node:path";
+import { isEnoent } from "../../util/fs.js";
 import { DIARY_CHUNK_CORPUS, indexDiaryMarkdown } from "./chunks.js";
 export const DIARY_INDEX_FILE_RE = /^\d{4}-\d{2}-\d{2}\.md$/;
 export function isDatedDiaryMarkdownFile(path) {
@@ -22,9 +23,6 @@ export async function listDiaryMarkdownFiles(config) {
         .map((entry) => join(config.memory.diariesDir, entry.name))
         .sort();
 }
-function isEnoent(error) {
-    return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
-}
 export async function indexDiaryFile(options) {
     const path = resolveDiaryPath(options.config, options.path);
     const sourceId = basename(path);

package/dist/memory/doctor.js

@@ -1,3 +1,4 @@
+import { runInTransaction } from "./util.js";
 export function runDoctor(stores, opts = {}) {
     void opts;
     const findings = [];
@@ -32,10 +33,7 @@ export function applyDoctorFixes(stores, report) {
             fixed += before;
         }
     };
-    if (stores.index.db.inTransaction)
-        runIndexFixes();
-    else
-        stores.index.db.transaction(runIndexFixes).immediate();
+    runInTransaction(stores.index.db, runIndexFixes);
     const runLcmFixes = () => {
         fixed += stores.lcm.db
             .prepare(`DELETE FROM lcm_segments
@@ -63,10 +61,7 @@ export function applyDoctorFixes(stores, report) {
             }
         }
     };
-    if (stores.lcm.db.inTransaction)
-        runLcmFixes();
-    else
-        stores.lcm.db.transaction(runLcmFixes).immediate();
+    runInTransaction(stores.lcm.db, runLcmFixes);
     if (report.findings.some((finding) => finding.kind === "summary_fk_violation")) {
         warnings.push("summary FK violations were not modified; inspect LCM summary lineage manually");
     }

package/dist/memory/index/chunk-indexer.js

@@ -99,10 +99,13 @@ export class ChunkIndexer {
         const toInsert = [];
         const insertPositions = [];
         const existingMappings = [];
+        const existingMappingIds = new Map();
+        const insertKnownMissing = new Set();
         for (let resultIndex = 0; resultIndex < prepared.length; resultIndex++) {
             const item = prepared[resultIndex];
             if (item.existingId !== null) {
                 ids[resultIndex] = item.existingId;
+                existingMappingIds.set(item.contentHash, item.existingId);
                 existingMappings.push({
                     corpus: item.input.corpus,
                     sourceId: item.sourceId,
@@ -120,6 +123,7 @@ export class ChunkIndexer {
             if (!embedding)
                 throw new Error("Missing embedding for memory chunk");
             insertPositions.push(resultIndex);
+            insertKnownMissing.add(item.contentHash);
             toInsert.push({
                 corpus: item.input.corpus,
                 sourceId: item.sourceId,
@@ -137,8 +141,8 @@ export class ChunkIndexer {
             if (replaceSource) {
                 this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, replaceSource.keepMappings);
             }
-            this.store.recordSourceMappings(existingMappings);
-            insertedIds = this.store.insertChunks(toInsert);
+            this.store.recordSourceMappings(existingMappings, existingMappingIds);
+            insertedIds = this.store.insertChunks(toInsert, undefined, insertKnownMissing);
         };
         if (replaceSource && !this.store.db.inTransaction)
             this.store.db.transaction(writeChunks).immediate();

package/dist/memory/index/retrieval.js

@@ -1,3 +1,4 @@
+import { positiveIntegerOrDefault } from "../util.js";
 const DEFAULT_LIMIT = 8;
 const DEFAULT_CANDIDATE_MULTIPLIER = 4;
 const RRF_K = 60;
@@ -241,6 +242,3 @@ function chunkSources(chunk) {
 function uniqueStrings(values) {
     return Array.from(new Set(values?.filter((value) => value.trim()) ?? []));
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/index/store.js

@@ -55,25 +55,31 @@ export class MemoryIndexStore {
     insertChunk(input) {
         return this.insertChunks([input])[0];
     }
-    insertChunks(inputs) {
+    insertChunks(inputs, preloadedIds, knownMissingHashes) {
         if (inputs.length === 0)
             return [];
         const rows = inputs.map((input) => this.normalizeInput(input));
+        const knownIds = new Map(preloadedIds);
         const out = [];
         const insert = this.db.transaction((items) => {
             for (const item of items)
-                out.push(this.insertNormalized(item));
+                out.push(this.insertNormalized(item, knownIds, knownMissingHashes));
         });
         insert.immediate(rows);
         return out;
     }
-    recordSourceMappings(inputs) {
+    recordSourceMappings(inputs, preloadedIds) {
         if (inputs.length === 0)
             return;
         const rows = inputs.map((input) => this.normalizeInput(input));
         this.db
             .transaction((items) => {
             for (const item of items) {
+                const preloadedId = preloadedIds?.get(item.contentHash);
+                if (preloadedId !== undefined) {
+                    this.insertSourceMapping(preloadedId, item);
+                    continue;
+                }
                 const existing = this.db
                     .prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
                     .get(item.contentHash);
@@ -88,8 +94,9 @@ export class MemoryIndexStore {
         const out = [];
         const replace = this.db.transaction(() => {
             this.deleteBySourceInternal(corpus, sourceId);
+            const knownIds = new Map();
             for (const item of rows)
-                out.push(this.insertNormalized(item));
+                out.push(this.insertNormalized(item, knownIds));
         });
         replace.immediate();
         return out;
@@ -168,19 +175,19 @@ export class MemoryIndexStore {
         return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
     }
     searchSemanticLinear(query, normalized) {
-        const rows = this.db
-            .prepare(normalized.corpus
+        const stmt = this.db.prepare(normalized.corpus
             ? `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.corpus = ?`
-            : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`)
-            .all(...(normalized.corpus ? [normalized.corpus] : []));
-        return rows
-            .map((row) => ({
-            id: row.id,
-            score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
-            chunk: rowToChunk(row),
-        }))
-            .sort((a, b) => a.score - b.score)
-            .slice(0, normalized.limit);
+            : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`);
+        const best = [];
+        for (const row of stmt.iterate(...(normalized.corpus ? [normalized.corpus] : []))) {
+            const hit = {
+                id: row.id,
+                score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
+                chunk: rowToChunk(row),
+            };
+            insertBoundedHit(best, hit, normalized.limit);
+        }
+        return best;
     }
     deleteChunk(id) {
         const remove = this.db.transaction(() => {
@@ -243,6 +250,7 @@ export class MemoryIndexStore {
             this.deleteBySource(corpus, sourceId);
             return;
         }
+        const keptHashes = new Set(kept.map((item) => item.contentHash));
         this.db
             .transaction(() => {
             const rows = this.db
@@ -259,7 +267,8 @@ export class MemoryIndexStore {
                 this.db
                     .prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
                     .run(corpus, sourceId, row.chunk_index);
-                this.deleteOrphanChunk(row.id);
+                if (!keptHashes.has(row.content_hash))
+                    this.deleteOrphanChunk(row.id);
             }
         })
             .immediate();
@@ -362,9 +371,15 @@ export class MemoryIndexStore {
             }),
         };
     }
-    insertNormalized(item) {
-        const existing = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
+    insertNormalized(item, knownIds, knownMissingHashes) {
+        const knownId = knownIds.get(item.contentHash);
+        const existing = knownId !== undefined
+            ? { id: knownId }
+            : knownMissingHashes?.has(item.contentHash)
+                ? undefined
+                : this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
         if (existing) {
+            knownIds.set(item.contentHash, existing.id);
             this.insertSourceMapping(existing.id, item);
             return existing.id;
         }
@@ -383,6 +398,7 @@ export class MemoryIndexStore {
                 .prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
                 .run(id, encodeVector(item.embedding));
         }
+        knownIds.set(item.contentHash, id);
         this.insertSourceMapping(id, item);
         return id;
     }
@@ -434,6 +450,18 @@ function normalizeSearchOptions(options) {
         corpus: options.corpus,
     };
 }
+function insertBoundedHit(best, hit, limit) {
+    if (limit <= 0)
+        return;
+    let index = best.findIndex((candidate) => hit.score < candidate.score);
+    if (index < 0)
+        index = best.length;
+    if (index >= limit)
+        return;
+    best.splice(index, 0, hit);
+    if (best.length > limit)
+        best.length = limit;
+}
 function sourcesJsonSelect(chunkIdExpr) {
     return `(SELECT json_group_array(json_object(
 		'corpus', s.corpus,

package/dist/memory/lcm/backfill.js

@@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises";
 import { relative, resolve } from "node:path";
 import { indexLcmRecords } from "./indexer.js";
 import { normalizeChatRecords } from "./normalize.js";
-import { computeLcmRecordKey } from "./store.js";
 const DEFAULT_YIELD_EVERY_N = 1024;
 const INDEX_BATCH_SIZE = 32;
 export async function backfillFromChatLogs(deps, options) {
@@ -66,15 +65,12 @@ export async function backfillFromChatLogs(deps, options) {
                 }
                 const inserted = [];
                 for (const record of batch.records) {
-                    if (recordExists(deps.lcmStore, record)) {
+                    const result = deps.lcmStore.insertRecordReturningStored(record);
+                    if (!result.inserted) {
                         report.recordsSkippedDuplicate += 1;
                         continue;
                     }
-                    const id = deps.lcmStore.insertRecord(record);
-                    const stored = deps.lcmStore.getRecord(id);
-                    if (!stored)
-                        throw new Error(`Failed to read backfilled LCM record: ${id}`);
-                    inserted.push(stored);
+                    inserted.push(result.record);
                     report.recordsInserted += 1;
                     if (inserted.length >= INDEX_BATCH_SIZE) {
                         report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
@@ -225,17 +221,24 @@ function countMissingSegments(lcmStore, segmentIds) {
     return missing;
 }
 function countExistingRecords(lcmStore, records) {
-    let existing = 0;
-    for (const record of records) {
-        if (recordExists(lcmStore, record))
-            existing += 1;
+    if (records.length === 0)
+        return 0;
+    const keys = records.map((record) => lcmStore.computeRecordKey(record));
+    const existingKeys = new Set();
+    for (const chunk of chunks([...new Set(keys)], 256)) {
+        const rows = lcmStore.db
+            .prepare(`SELECT record_key FROM lcm_records WHERE record_key IN (${chunk.map(() => "?").join(",")})`)
+            .all(...chunk);
+        for (const row of rows)
+            existingKeys.add(row.record_key);
     }
-    return existing;
+    return keys.reduce((total, key) => total + (existingKeys.has(key) ? 1 : 0), 0);
 }
-function recordExists(lcmStore, record) {
-    return !!lcmStore.db
-        .prepare("SELECT 1 FROM lcm_records WHERE record_key = ? LIMIT 1")
-        .get(computeLcmRecordKey(record));
+function chunks(items, size) {
+    const out = [];
+    for (let index = 0; index < items.length; index += size)
+        out.push(items.slice(index, index + size));
+    return out;
 }
 function errorCode(error) {
     return error && typeof error === "object" && "code" in error

package/dist/memory/lcm/context-transformer.js

@@ -1,5 +1,5 @@
 import { condense } from "./condense.js";
-import { createRawContextItems, estimateAgentMessageTokens, lcmRecordToAgentMessage, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
+import { createRawContextItems, estimateAgentMessageTokens, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
 import { indexLcmSummaries } from "./indexer.js";
 import { createSyntheticLcmSummaryMessage } from "./summarizer.js";
 const LCM_SUMMARY_OPEN_TAG = "<from_earlier>";
@@ -48,6 +48,7 @@ export class LcmContextTransformer {
                     signal,
                     model: options.model,
                     promptText,
+                    initialPressure: pressure,
                 });
             }
         }
@@ -76,7 +77,9 @@ export class LcmContextTransformer {
     }
     async serviceCompactionDebtForState(input) {
         for (let round = 0; input.state.compactionDebt > 0 && round < this.settings.maxRounds; round += 1) {
-            const pressure = this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
+            const pressure = round === 0 && input.initialPressure
+                ? input.initialPressure
+                : this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
             if (!pressure.candidate.shouldCompact) {
                 if (pressure.thresholdOverflowTokens > 0) {
                     const condensed = await this.condenseRuntimeSummaries({
@@ -262,8 +265,9 @@ export class LcmContextTransformer {
         this.lcmStore.db
             .transaction(() => {
             for (const insert of inserts) {
-                insert.item.recordId = this.lcmStore.insertRecord(insert.input);
-                insert.item.record = this.lcmStore.getRecord(insert.item.recordId);
+                const result = this.lcmStore.insertRecordReturningStored(insert.input);
+                insert.item.recordId = result.record.id;
+                insert.item.record = result.record;
             }
         })
             .immediate();
@@ -275,20 +279,7 @@ export class LcmContextTransformer {
         const items = [];
         for (const row of rows) {
             if (row.type === "raw") {
-                const record = this.lcmStore.getRecord(row.recordId);
-                if (!record) {
-                    console.error(`memory LCM context item dropped because record ${row.recordId} is missing`);
-                    continue;
-                }
-                const message = lcmRecordToAgentMessage(record);
-                items.push({
-                    type: "raw",
-                    id: row.fingerprint,
-                    recordId: record.id,
-                    record,
-                    message,
-                    tokens: estimateAgentMessageTokens(message),
-                });
+                // Raw history is owned by transcripts; legacy raw context rows must not replay it.
                 continue;
             }
             const summary = this.lcmStore.getSummary(row.summaryId);
@@ -359,8 +350,6 @@ function syncContextState(state, messages) {
         const replacement = rawById.get(item.id);
         if (replacement && !covered.has(item.id))
             next.push(replacement);
-        else if (item.recordId !== null && !covered.has(item.id))
-            next.push(item);
     }
     for (const item of rawItems) {
         if (!covered.has(item.id) && !next.some((existing) => existing.type === "raw" && existing.id === item.id)) {
@@ -390,8 +379,6 @@ function contextItemsForStorage(items) {
         const timestamp = item.message.timestamp;
         const happenedAt = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
         if (item.type === "raw") {
-            if (item.recordId !== null)
-                stored.push({ type: "raw", recordId: item.recordId, fingerprint: item.id, happenedAt });
             continue;
         }
         if (item.persistedSummaryId !== undefined) {
@@ -522,9 +509,10 @@ function assembleWithinBudget(state, settings, model) {
     const freshTail = state.items.slice(resolveFreshTailStartIndexForState(state.items, settings));
     const selected = new Set(freshTail);
     let tokens = sumItemTokens(freshTail);
+    const originalIndexes = new Map(state.items.map((item, index) => [item, index]));
     const summaries = state.items
         .filter((item) => item.type === "summary" && !selected.has(item))
-        .sort((a, b) => b.depth - a.depth || state.items.indexOf(b) - state.items.indexOf(a));
+        .sort((a, b) => b.depth - a.depth || (originalIndexes.get(b) ?? 0) - (originalIndexes.get(a) ?? 0));
     for (const item of summaries) {
         if (tokens + item.tokens > budget && selected.size > 0)
             continue;
@@ -587,7 +575,7 @@ function lcmRecordPartsFromAgentMessage(message) {
                 kind: "tool_result",
                 toolCallId: toolResult.toolCallId ?? "",
                 toolName: toolResult.toolName ?? "tool",
-                output: toolResult.details ?? textFromContent(toolResult.content),
+                output: textFromContent(toolResult.content),
                 ...(toolResult.isError ? { isError: true } : {}),
             },
         ];

package/dist/memory/lcm/context.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { scoreEvictable, tokenBag } from "./eviction-score.js";
+import { buildEvictionScoreContext, scoreEvictable, tokenBag } from "./eviction-score.js";
 const MESSAGE_OVERHEAD_TOKENS = 6;
 const RECORD_OVERHEAD_TOKENS = 4;
 const IMAGE_TOKEN_ESTIMATE = 1200;
@@ -167,9 +167,12 @@ function selectLeafChunk(items, leafChunkTokens, promptText, config) {
     const records = items.map((item) => item.record).filter((record) => !!record);
     if (records.length === 0)
         return selectOldestLeafChunk(items, leafChunkTokens);
+    const scoreContext = buildEvictionScoreContext(promptText, records);
+    if (!scoreContext)
+        return selectOldestLeafChunk(items, leafChunkTokens);
     const scored = targetRanges.map((range) => ({
         ...range,
-        score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records) : 0), 0),
+        score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records, scoreContext) : 0), 0),
     }));
     scored.sort((a, b) => a.score - b.score || a.startIndex - b.startIndex);
     return scored[0]?.items ?? [];
@@ -179,7 +182,7 @@ function createValidLeafRanges(items, leafChunkTokens) {
     for (let startIndex = 0; startIndex < items.length; startIndex += 1) {
         if (isToolResultContinuingPreviousToolCall(items, startIndex))
             continue;
-        const chunk = selectOldestLeafChunk(items.slice(startIndex), leafChunkTokens);
+        const chunk = selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens);
         if (chunk.length === 0)
             continue;
         const chunkTokens = chunk.reduce((total, item) => total + item.tokens, 0);
@@ -194,9 +197,12 @@ function createValidLeafRanges(items, leafChunkTokens) {
     return ranges;
 }
 function selectOldestLeafChunk(items, leafChunkTokens) {
+    return selectOldestLeafChunkFromIndex(items, 0, leafChunkTokens);
+}
+function selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens) {
     const chunk = [];
     let tokens = 0;
-    for (let index = 0; index < items.length; index += 1) {
+    for (let index = startIndex; index < items.length; index += 1) {
         const item = items[index];
         if (!item)
             continue;

package/dist/memory/lcm/eviction-score.js

@@ -4,34 +4,46 @@ export function tokenBag(text) {
         .split(/[^a-z0-9]+/)
         .filter((token) => token.length >= 2);
 }
-export function scoreEvictable(record, prompt, allRecords) {
+export function buildEvictionScoreContext(prompt, allRecords) {
     const promptTerms = tokenBag(prompt);
     if (promptTerms.length === 0)
-        return 0;
-    const recordTerms = tokenBag(record.text);
-    if (recordTerms.length === 0)
-        return 0;
-    const recordFreq = new Map();
-    for (const term of recordTerms)
-        recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
+        return null;
     const promptUniqueTerms = new Set(promptTerms);
     const documentFrequencies = new Map();
+    const recordTerms = new Map();
     for (const candidate of allRecords) {
-        const candidateTerms = new Set(tokenBag(candidate.text));
+        const terms = tokenBag(candidate.text);
+        recordTerms.set(candidate, terms);
+        const candidateTerms = new Set(terms);
         for (const term of promptUniqueTerms) {
             if (candidateTerms.has(term))
                 documentFrequencies.set(term, (documentFrequencies.get(term) ?? 0) + 1);
         }
     }
-    const candidateCount = Math.max(0, allRecords.length);
+    return {
+        promptUniqueTerms,
+        documentFrequencies,
+        recordTerms,
+        candidateCount: Math.max(0, allRecords.length),
+    };
+}
+export function scoreEvictable(record, prompt, allRecords, context = buildEvictionScoreContext(prompt, allRecords)) {
+    if (!context)
+        return 0;
+    const recordTerms = context.recordTerms.get(record) ?? tokenBag(record.text);
+    if (recordTerms.length === 0)
+        return 0;
+    const recordFreq = new Map();
+    for (const term of recordTerms)
+        recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
     let score = 0;
-    for (const term of promptUniqueTerms) {
+    for (const term of context.promptUniqueTerms) {
         const tf = recordFreq.get(term) ?? 0;
         if (tf <= 0)
             continue;
         const normalizedTf = tf / recordTerms.length;
-        const df = documentFrequencies.get(term) ?? 0;
-        const idf = Math.log((candidateCount + 1) / (df + 1) + 1);
+        const df = context.documentFrequencies.get(term) ?? 0;
+        const idf = Math.log((context.candidateCount + 1) / (df + 1) + 1);
         score += normalizedTf * idf;
     }
     return score;