@qearlyao/familiar 0.2.3 → 0.2.4

package/dist/image-gen.js

@@ -1,21 +1,15 @@
 import { randomUUID } from "node:crypto";
 import { lstat, writeFile } from "node:fs/promises";
-import { basename, extname, isAbsolute, relative, resolve } from "node:path";
+import { basename, isAbsolute, relative, resolve } from "node:path";
 import { findEnvKeys, generateImages, getEnvApiKey, getImageModels, getImageProviders, } from "@earendil-works/pi-ai";
 import { Type } from "typebox";
 import { ensureGeneratedAttachmentsDir } from "./generated-media.js";
 import { ensureInlineImageDerivative } from "./image-derivatives.js";
 import { promptImagesFromAttachments } from "./inbound-attachments.js";
 import { parseModelRef } from "./models.js";
+import { imageMimeTypeFromPath, sniffImageMimeType } from "./util/image-mime.js";
 const IMAGE_GEN_NOTICE_PREFIX = "Generated image attachment:";
 const OPENROUTER_IMAGE_BASE_URL = "https://openrouter.ai/api/v1";
-const IMAGE_MIME_BY_EXTENSION = {
-    ".jpg": "image/jpeg",
-    ".jpeg": "image/jpeg",
-    ".png": "image/png",
-    ".gif": "image/gif",
-    ".webp": "image/webp",
-};
 const imageGenSchema = Type.Object({
     prompt: Type.String({ description: "Image generation prompt." }),
     referenceImages: Type.Optional(Type.Array(Type.String(), {
@@ -107,20 +101,6 @@ function textOutput(result) {
         .filter(Boolean)
         .join("\n");
 }
-function imageMimeTypeFromBytes(buffer) {
-    if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
-        return "image/jpeg";
-    if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
-        return "image/png";
-    }
-    if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
-        return "image/gif";
-    }
-    if (buffer.subarray(0, 4).toString("ascii") === "RIFF" && buffer.subarray(8, 12).toString("ascii") === "WEBP") {
-        return "image/webp";
-    }
-    return undefined;
-}
 function recoveredImageFromBase64(value) {
     const data = value.trim();
     if (!/^[A-Za-z0-9+/]+={0,2}$/.test(data) || data.length % 4 !== 0)
@@ -128,7 +108,7 @@ function recoveredImageFromBase64(value) {
     const buffer = Buffer.from(data, "base64");
     if (!buffer.length)
         return undefined;
-    const detectedMimeType = imageMimeTypeFromBytes(buffer);
+    const detectedMimeType = sniffImageMimeType(buffer);
     if (!detectedMimeType)
         return undefined;
     return {
@@ -167,9 +147,6 @@ function normalizeCompatibleImageText(result) {
         return result;
     return { ...result, output };
 }
-function mimeTypeFromPath(path) {
-    return IMAGE_MIME_BY_EXTENSION[extname(path).toLowerCase()];
-}
 function resolveWorkspaceReferencePath(config, rawRef) {
     const path = isAbsolute(rawRef) ? resolve(rawRef) : resolve(config.workspacePath, rawRef);
     const workspaceRelative = relative(config.workspacePath, path);
@@ -190,7 +167,7 @@ async function collectWorkspaceReferenceImages(config, rawRef) {
     }
     if (!pathStat.isFile())
         throw new Error(`Reference image path is not a file or folder: ${rawRef}`);
-    const mimeType = mimeTypeFromPath(path);
+    const mimeType = imageMimeTypeFromPath(path);
     if (!mimeType)
         throw new Error(`Reference image path is not a supported image: ${rawRef}`);
     return [
@@ -338,11 +315,7 @@ async function tryGenerateImages(config, ref, prompt, references, workspaceRefs,
 }
 function attemptDetails(model, result) {
     return {
-        provider: model.provider,
-        model: model.id,
-        api: model.api,
-        baseUrl: model.baseUrl,
-        ...(result.responseId ? { responseId: result.responseId } : {}),
+        model: `${model.provider}/${model.id}`,
         stopReason: result.stopReason,
         ...(result.errorMessage ? { errorMessage: result.errorMessage } : {}),
     };
@@ -378,18 +351,8 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
                 }
                 catch (error) {
                     const message = error instanceof Error ? error.message : String(error);
-                    let baseUrl = "";
-                    try {
-                        baseUrl = resolveImageModel(config, ref).baseUrl;
-                    }
-                    catch {
-                        baseUrl = "";
-                    }
                     attempts.push({
-                        provider: ref.provider,
-                        model: ref.id,
-                        api: config.imageGen.api,
-                        baseUrl,
+                        model: `${ref.provider}/${ref.id}`,
                         stopReason: "error",
                         errorMessage: message,
                     });
@@ -414,8 +377,10 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
             if (!selected)
                 throw new Error(`Image generation failed: ${selectedError}`);
             const attachments = await writeGeneratedImages(config, mediaSink, selected.result);
+            const primaryAttachment = attachments[0];
             const notices = attachments.map((attachment) => formatImageGenNotice(attachment.name));
             const sideText = textOutput(selected.result);
+            const selectedAttempt = attempts.at(-1);
             return {
                 content: [
                     {
@@ -424,15 +389,11 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
                     },
                 ],
                 details: {
-                    provider: selected.model.provider,
-                    model: selected.model.id,
-                    api: selected.model.api,
-                    baseUrl: selected.model.baseUrl,
-                    prompt,
-                    ...(selected.result.responseId ? { responseId: selected.result.responseId } : {}),
+                    model: `${selected.model.provider}/${selected.model.id}`,
                     ...(sideText ? { textOutput: sideText } : {}),
-                    attachments,
-                    attempts,
+                    ...(primaryAttachment ? { id: primaryAttachment.id, localPath: primaryAttachment.localPath } : {}),
+                    stopReason: selectedAttempt?.stopReason ?? selected.result.stopReason,
+                    ...(selectedAttempt?.errorMessage ? { errorMessage: selectedAttempt.errorMessage } : {}),
                 },
             };
         },

package/dist/inbound-attachments.js

@@ -4,10 +4,13 @@ import { basename, extname, resolve } from "node:path";
 import { attachmentsDir, publicAttachmentPath } from "./generated-media.js";
 import { ensureInlineImageDerivative, MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
 import { deriveInboundAttachmentText } from "./media-understanding.js";
+import { IMAGE_EXTENSION_BY_MIME, sniffImageMimeType } from "./util/image-mime.js";
 export { MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
 export const MAX_INBOUND_ATTACHMENTS = 4;
 export const MAX_INBOUND_ATTACHMENT_BYTES = 12 * 1024 * 1024;
 export const MAX_INBOUND_TOTAL_BYTES = 24 * 1024 * 1024;
+const TEXT_ATTACHMENT_PREVIEW_LINES = 2;
+const TEXT_ATTACHMENT_PREVIEW_CHARS = 1000;
 const ALLOWED_MIME_TYPES = new Set([
     "image/jpeg",
     "image/png",
@@ -23,10 +26,7 @@ const ALLOWED_MIME_TYPES = new Set([
     "text/plain",
 ]);
 const EXTENSIONS_BY_MIME = {
-    "image/jpeg": ".jpg",
-    "image/png": ".png",
-    "image/gif": ".gif",
-    "image/webp": ".webp",
+    ...IMAGE_EXTENSION_BY_MIME,
     "audio/mpeg": ".mp3",
     "audio/ogg": ".ogg",
     "audio/wav": ".wav",
@@ -51,32 +51,42 @@ function kindFromMime(mimeType) {
         return "video";
     return "file";
 }
+function textAttachmentPreview(buffer, mimeType) {
+    if (mimeType !== "text/plain")
+        return undefined;
+    const decoded = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
+    const normalized = decoded.replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
+    if (!normalized || normalized.includes("\uFFFD"))
+        return undefined;
+    const lines = normalized.split("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_LINES);
+    const preview = lines.join("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_CHARS).trim();
+    return preview || undefined;
+}
 function sniffText(buffer) {
     if (buffer.length === 0)
         return "text/plain";
     const head = buffer.subarray(0, Math.min(buffer.length, 512));
     if (head.includes(0))
         return undefined;
-    return head.every((byte) => byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126))
-        ? "text/plain"
-        : undefined;
-}
-function sniffMimeType(buffer, declared) {
-    let detected;
-    if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
-        detected = "image/jpeg";
-    else if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
-        detected = "image/png";
+    let decoded;
+    try {
+        decoded = new TextDecoder("utf-8", { fatal: true }).decode(head);
     }
-    else if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" ||
-        buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
-        detected = "image/gif";
+    catch {
+        return undefined;
     }
-    else if (buffer.subarray(0, 4).toString("ascii") === "RIFF" &&
-        buffer.subarray(8, 12).toString("ascii") === "WEBP") {
-        detected = "image/webp";
+    for (const char of decoded) {
+        const code = char.codePointAt(0) ?? 0;
+        if (code === 9 || code === 10 || code === 13)
+            continue;
+        if (code < 32 || code === 127)
+            return undefined;
     }
-    else if (buffer.subarray(0, 4).toString("ascii") === "%PDF")
+    return "text/plain";
+}
+function sniffMimeType(buffer, declared) {
+    let detected = sniffImageMimeType(buffer);
+    if (!detected && buffer.subarray(0, 4).toString("ascii") === "%PDF")
         detected = "application/pdf";
     else if (buffer.subarray(0, 3).toString("ascii") === "ID3" ||
         buffer.subarray(0, 2).equals(Buffer.from([0xff, 0xfb]))) {
@@ -204,6 +214,18 @@ export async function materializeInboundAttachments(config, inputs) {
                 source: attachment.source,
                 sha256: attachment.sha256,
             };
+            const textPreview = textAttachmentPreview(attachment.buffer, attachment.mimeType);
+            if (textPreview) {
+                finalAttachment.derived = {
+                    ...finalAttachment.derived,
+                    text: {
+                        provider: "local",
+                        model: "text-preview",
+                        label: "preview",
+                        text: textPreview,
+                    },
+                };
+            }
             const derivedImage = await ensureInlineImageDerivative(config, finalAttachment);
             if (derivedImage) {
                 if (derivedImage.localPath && !existingDerivedPaths.has(derivedImage.localPath)) {
@@ -260,7 +282,16 @@ export async function promptImagesFromAttachments(attachments) {
 export function promptAttachmentNotes(attachments) {
     return attachments
         .map((attachment) => {
-        const attrs = `name="${attachment.name}" id="${attachment.id}" kind="${attachment.kind ?? "file"}" mime="${attachment.mimeType ?? "unknown"}" size="${attachment.size ?? "unknown"}"`;
+        const attrs = [
+            `name="${attachment.name}"`,
+            `id="${attachment.id}"`,
+            `kind="${attachment.kind ?? "file"}"`,
+            `mime="${attachment.mimeType ?? "unknown"}"`,
+            `size="${attachment.size ?? "unknown"}"`,
+            attachment.localPath ? `path="${attachment.localPath}"` : undefined,
+        ]
+            .filter(Boolean)
+            .join(" ");
         const derivedText = attachment.derived?.text?.text;
         if (derivedText) {
             const label = attachment.derived?.text?.label || (attachment.kind === "audio" ? "transcription" : "summary");

package/dist/memory/diary/ambient-injector.js

@@ -1,3 +1,4 @@
+import { positiveIntegerOrDefault } from "../util.js";
 import { retrieveAmbientDiary } from "./ambient.js";
 const INJECTED_MEMORY_OPEN = "<injected_memory>";
 const INJECTED_MEMORY_CLOSE = "</injected_memory>";
@@ -64,9 +65,6 @@ export class AmbientDiaryInjector {
         }
     }
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}
 function nonNegativeIntegerOrDefault(value, fallback) {
     return value !== undefined && Number.isInteger(value) && value >= 0 ? value : fallback;
 }

package/dist/memory/diary/ambient.js

@@ -1,4 +1,5 @@
 import { retrieveMemory, } from "../index/retrieval.js";
+import { positiveIntegerOrDefault } from "../util.js";
 import { DIARY_CHUNK_CORPUS } from "./chunks.js";
 const DEFAULT_LIMIT = 4;
 const DEFAULT_CANDIDATE_MULTIPLIER = 5;
@@ -119,6 +120,3 @@ function normalizeUnit(value) {
     const absolute = Math.abs(value);
     return Math.max(0, Math.min(1, absolute > 1 ? absolute / 10 : absolute));
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/diary/chunks.js

@@ -1,4 +1,5 @@
 import { basename } from "node:path";
+import { positiveIntegerOrDefault } from "../util.js";
 export const DIARY_CHUNK_CORPUS = "diary_chunk";
 const DEFAULT_MAX_CHARS = 2400;
 const DIARY_DATE_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
@@ -226,6 +227,3 @@ function stripInlineMarkdown(value) {
 function isMarkdownHeading(line) {
     return /^#{1,6}\s+/.test(line);
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/diary/indexer.js

@@ -1,5 +1,6 @@
 import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, isAbsolute, join, resolve } from "node:path";
+import { isEnoent } from "../../util/fs.js";
 import { DIARY_CHUNK_CORPUS, indexDiaryMarkdown } from "./chunks.js";
 export const DIARY_INDEX_FILE_RE = /^\d{4}-\d{2}-\d{2}\.md$/;
 export function isDatedDiaryMarkdownFile(path) {
@@ -22,9 +23,6 @@ export async function listDiaryMarkdownFiles(config) {
         .map((entry) => join(config.memory.diariesDir, entry.name))
         .sort();
 }
-function isEnoent(error) {
-    return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
-}
 export async function indexDiaryFile(options) {
     const path = resolveDiaryPath(options.config, options.path);
     const sourceId = basename(path);

package/dist/memory/doctor.js

@@ -1,3 +1,4 @@
+import { runInTransaction } from "./util.js";
 export function runDoctor(stores, opts = {}) {
     void opts;
     const findings = [];
@@ -32,10 +33,7 @@ export function applyDoctorFixes(stores, report) {
             fixed += before;
         }
     };
-    if (stores.index.db.inTransaction)
-        runIndexFixes();
-    else
-        stores.index.db.transaction(runIndexFixes).immediate();
+    runInTransaction(stores.index.db, runIndexFixes);
     const runLcmFixes = () => {
         fixed += stores.lcm.db
             .prepare(`DELETE FROM lcm_segments
@@ -63,10 +61,7 @@ export function applyDoctorFixes(stores, report) {
             }
         }
     };
-    if (stores.lcm.db.inTransaction)
-        runLcmFixes();
-    else
-        stores.lcm.db.transaction(runLcmFixes).immediate();
+    runInTransaction(stores.lcm.db, runLcmFixes);
     if (report.findings.some((finding) => finding.kind === "summary_fk_violation")) {
         warnings.push("summary FK violations were not modified; inspect LCM summary lineage manually");
     }

package/dist/memory/index/chunk-indexer.js

@@ -99,10 +99,13 @@ export class ChunkIndexer {
         const toInsert = [];
         const insertPositions = [];
         const existingMappings = [];
+        const existingMappingIds = new Map();
+        const insertKnownMissing = new Set();
         for (let resultIndex = 0; resultIndex < prepared.length; resultIndex++) {
             const item = prepared[resultIndex];
             if (item.existingId !== null) {
                 ids[resultIndex] = item.existingId;
+                existingMappingIds.set(item.contentHash, item.existingId);
                 existingMappings.push({
                     corpus: item.input.corpus,
                     sourceId: item.sourceId,
@@ -120,6 +123,7 @@ export class ChunkIndexer {
             if (!embedding)
                 throw new Error("Missing embedding for memory chunk");
             insertPositions.push(resultIndex);
+            insertKnownMissing.add(item.contentHash);
             toInsert.push({
                 corpus: item.input.corpus,
                 sourceId: item.sourceId,
@@ -137,8 +141,8 @@ export class ChunkIndexer {
             if (replaceSource) {
                 this.store.deleteBySourceExceptMappings(replaceSource.corpus, replaceSource.sourceId, replaceSource.keepMappings);
             }
-            this.store.recordSourceMappings(existingMappings);
-            insertedIds = this.store.insertChunks(toInsert);
+            this.store.recordSourceMappings(existingMappings, existingMappingIds);
+            insertedIds = this.store.insertChunks(toInsert, undefined, insertKnownMissing);
         };
         if (replaceSource && !this.store.db.inTransaction)
             this.store.db.transaction(writeChunks).immediate();

package/dist/memory/index/retrieval.js

@@ -1,3 +1,4 @@
+import { positiveIntegerOrDefault } from "../util.js";
 const DEFAULT_LIMIT = 8;
 const DEFAULT_CANDIDATE_MULTIPLIER = 4;
 const RRF_K = 60;
@@ -241,6 +242,3 @@ function chunkSources(chunk) {
 function uniqueStrings(values) {
     return Array.from(new Set(values?.filter((value) => value.trim()) ?? []));
 }
-function positiveIntegerOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
-}

package/dist/memory/index/store.js

@@ -55,25 +55,31 @@ export class MemoryIndexStore {
     insertChunk(input) {
         return this.insertChunks([input])[0];
     }
-    insertChunks(inputs) {
+    insertChunks(inputs, preloadedIds, knownMissingHashes) {
         if (inputs.length === 0)
             return [];
         const rows = inputs.map((input) => this.normalizeInput(input));
+        const knownIds = new Map(preloadedIds);
         const out = [];
         const insert = this.db.transaction((items) => {
             for (const item of items)
-                out.push(this.insertNormalized(item));
+                out.push(this.insertNormalized(item, knownIds, knownMissingHashes));
         });
         insert.immediate(rows);
         return out;
     }
-    recordSourceMappings(inputs) {
+    recordSourceMappings(inputs, preloadedIds) {
         if (inputs.length === 0)
             return;
         const rows = inputs.map((input) => this.normalizeInput(input));
         this.db
             .transaction((items) => {
             for (const item of items) {
+                const preloadedId = preloadedIds?.get(item.contentHash);
+                if (preloadedId !== undefined) {
+                    this.insertSourceMapping(preloadedId, item);
+                    continue;
+                }
                 const existing = this.db
                     .prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
                     .get(item.contentHash);
@@ -88,8 +94,9 @@ export class MemoryIndexStore {
         const out = [];
         const replace = this.db.transaction(() => {
             this.deleteBySourceInternal(corpus, sourceId);
+            const knownIds = new Map();
             for (const item of rows)
-                out.push(this.insertNormalized(item));
+                out.push(this.insertNormalized(item, knownIds));
         });
         replace.immediate();
         return out;
@@ -168,19 +175,19 @@ export class MemoryIndexStore {
         return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
     }
     searchSemanticLinear(query, normalized) {
-        const rows = this.db
-            .prepare(normalized.corpus
+        const stmt = this.db.prepare(normalized.corpus
             ? `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.corpus = ?`
-            : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`)
-            .all(...(normalized.corpus ? [normalized.corpus] : []));
-        return rows
-            .map((row) => ({
-            id: row.id,
-            score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
-            chunk: rowToChunk(row),
-        }))
-            .sort((a, b) => a.score - b.score)
-            .slice(0, normalized.limit);
+            : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`);
+        const best = [];
+        for (const row of stmt.iterate(...(normalized.corpus ? [normalized.corpus] : []))) {
+            const hit = {
+                id: row.id,
+                score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
+                chunk: rowToChunk(row),
+            };
+            insertBoundedHit(best, hit, normalized.limit);
+        }
+        return best;
     }
     deleteChunk(id) {
         const remove = this.db.transaction(() => {
@@ -243,6 +250,7 @@ export class MemoryIndexStore {
             this.deleteBySource(corpus, sourceId);
             return;
         }
+        const keptHashes = new Set(kept.map((item) => item.contentHash));
         this.db
             .transaction(() => {
             const rows = this.db
@@ -259,7 +267,8 @@ export class MemoryIndexStore {
                 this.db
                     .prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
                     .run(corpus, sourceId, row.chunk_index);
-                this.deleteOrphanChunk(row.id);
+                if (!keptHashes.has(row.content_hash))
+                    this.deleteOrphanChunk(row.id);
             }
         })
             .immediate();
@@ -362,9 +371,15 @@ export class MemoryIndexStore {
             }),
         };
     }
-    insertNormalized(item) {
-        const existing = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
+    insertNormalized(item, knownIds, knownMissingHashes) {
+        const knownId = knownIds.get(item.contentHash);
+        const existing = knownId !== undefined
+            ? { id: knownId }
+            : knownMissingHashes?.has(item.contentHash)
+                ? undefined
+                : this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
         if (existing) {
+            knownIds.set(item.contentHash, existing.id);
             this.insertSourceMapping(existing.id, item);
             return existing.id;
         }
@@ -383,6 +398,7 @@ export class MemoryIndexStore {
                 .prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
                 .run(id, encodeVector(item.embedding));
         }
+        knownIds.set(item.contentHash, id);
         this.insertSourceMapping(id, item);
         return id;
     }
@@ -434,6 +450,18 @@ function normalizeSearchOptions(options) {
         corpus: options.corpus,
     };
 }
+function insertBoundedHit(best, hit, limit) {
+    if (limit <= 0)
+        return;
+    let index = best.findIndex((candidate) => hit.score < candidate.score);
+    if (index < 0)
+        index = best.length;
+    if (index >= limit)
+        return;
+    best.splice(index, 0, hit);
+    if (best.length > limit)
+        best.length = limit;
+}
 function sourcesJsonSelect(chunkIdExpr) {
     return `(SELECT json_group_array(json_object(
 		'corpus', s.corpus,

package/dist/memory/lcm/backfill.js

@@ -2,7 +2,6 @@ import { readdir, readFile } from "node:fs/promises";
 import { relative, resolve } from "node:path";
 import { indexLcmRecords } from "./indexer.js";
 import { normalizeChatRecords } from "./normalize.js";
-import { computeLcmRecordKey } from "./store.js";
 const DEFAULT_YIELD_EVERY_N = 1024;
 const INDEX_BATCH_SIZE = 32;
 export async function backfillFromChatLogs(deps, options) {
@@ -66,15 +65,12 @@ export async function backfillFromChatLogs(deps, options) {
                 }
                 const inserted = [];
                 for (const record of batch.records) {
-                    if (recordExists(deps.lcmStore, record)) {
+                    const result = deps.lcmStore.insertRecordReturningStored(record);
+                    if (!result.inserted) {
                         report.recordsSkippedDuplicate += 1;
                         continue;
                     }
-                    const id = deps.lcmStore.insertRecord(record);
-                    const stored = deps.lcmStore.getRecord(id);
-                    if (!stored)
-                        throw new Error(`Failed to read backfilled LCM record: ${id}`);
-                    inserted.push(stored);
+                    inserted.push(result.record);
                     report.recordsInserted += 1;
                     if (inserted.length >= INDEX_BATCH_SIZE) {
                         report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
@@ -225,17 +221,24 @@ function countMissingSegments(lcmStore, segmentIds) {
     return missing;
 }
 function countExistingRecords(lcmStore, records) {
-    let existing = 0;
-    for (const record of records) {
-        if (recordExists(lcmStore, record))
-            existing += 1;
+    if (records.length === 0)
+        return 0;
+    const keys = records.map((record) => lcmStore.computeRecordKey(record));
+    const existingKeys = new Set();
+    for (const chunk of chunks([...new Set(keys)], 256)) {
+        const rows = lcmStore.db
+            .prepare(`SELECT record_key FROM lcm_records WHERE record_key IN (${chunk.map(() => "?").join(",")})`)
+            .all(...chunk);
+        for (const row of rows)
+            existingKeys.add(row.record_key);
     }
-    return existing;
+    return keys.reduce((total, key) => total + (existingKeys.has(key) ? 1 : 0), 0);
 }
-function recordExists(lcmStore, record) {
-    return !!lcmStore.db
-        .prepare("SELECT 1 FROM lcm_records WHERE record_key = ? LIMIT 1")
-        .get(computeLcmRecordKey(record));
+function chunks(items, size) {
+    const out = [];
+    for (let index = 0; index < items.length; index += size)
+        out.push(items.slice(index, index + size));
+    return out;
 }
 function errorCode(error) {
     return error && typeof error === "object" && "code" in error