@qearlyao/familiar 0.1.0

package/dist/memory/doctor.js

@@ -0,0 +1,250 @@
+export function runDoctor(stores, opts = {}) {
+    void opts;
+    const findings = [];
+    findDanglingIndexSources(stores, findings);
+    findOrphanEmptySegments(stores, findings);
+    findStaleLcmIndexRows(stores, findings);
+    findBrokenContextOrdering(stores, findings);
+    findSummaryFkViolations(stores, findings);
+    findMissingPrunedSnapshots(stores, findings);
+    findRequiresReindex(stores, findings);
+    findEmbeddingMismatches(stores, findings);
+    return { findings, clean: findings.length === 0 };
+}
+export function applyDoctorFixes(stores, report) {
+    let fixed = 0;
+    const warnings = [];
+    const runIndexFixes = () => {
+        fixed += stores.index.db
+            .prepare(`DELETE FROM memory_index_sources
+				 WHERE chunk_id NOT IN (SELECT id FROM memory_chunks)`)
+            .run().changes;
+        const staleSources = stores.index.db
+            .prepare(`SELECT corpus, source_id
+				 FROM memory_index_sources
+				 WHERE corpus IN ('lcm_record', 'lcm_summary')`)
+            .all();
+        for (const source of staleSources) {
+            if (lcmSourceExists(stores, source.corpus, source.source_id))
+                continue;
+            const before = countIndexSourceRows(stores.index, source.corpus, source.source_id);
+            stores.index.deleteBySourceUnsafe(source.corpus, source.source_id);
+            fixed += before;
+        }
+    };
+    if (stores.index.db.inTransaction)
+        runIndexFixes();
+    else
+        stores.index.db.transaction(runIndexFixes).immediate();
+    const runLcmFixes = () => {
+        fixed += stores.lcm.db
+            .prepare(`DELETE FROM lcm_segments
+				 WHERE status != 'active'
+				   AND id NOT IN (SELECT DISTINCT segment_id FROM lcm_records)
+				   AND id NOT IN (SELECT DISTINCT segment_id FROM lcm_summaries)`)
+            .run().changes;
+        const sessions = stores.lcm.db
+            .prepare("SELECT DISTINCT session_key FROM lcm_context_items ORDER BY session_key")
+            .all();
+        for (const session of sessions) {
+            const rows = stores.lcm.db
+                .prepare(`SELECT rowid AS rowid, ordinal
+					 FROM lcm_context_items
+					 WHERE session_key = ?
+					 ORDER BY ordinal, rowid`)
+                .all(session.session_key);
+            for (const [index, row] of rows.entries()) {
+                if (row.ordinal === index)
+                    continue;
+                stores.lcm.db
+                    .prepare("UPDATE lcm_context_items SET ordinal = ?, updated_at = unixepoch() WHERE rowid = ?")
+                    .run(index, row.rowid);
+                fixed += 1;
+            }
+        }
+    };
+    if (stores.lcm.db.inTransaction)
+        runLcmFixes();
+    else
+        stores.lcm.db.transaction(runLcmFixes).immediate();
+    if (report.findings.some((finding) => finding.kind === "summary_fk_violation")) {
+        warnings.push("summary FK violations were not modified; inspect LCM summary lineage manually");
+    }
+    if (report.findings.some((finding) => finding.kind === "missing_pruned_summary_snapshot")) {
+        warnings.push("missing pruned summary snapshots were not modified; inspect retained summaries manually");
+    }
+    if (report.findings.some((finding) => finding.kind === "embedding_mismatch")) {
+        warnings.push("embedding mismatches were not rebuilt; run 'familiar memory reindex'");
+    }
+    if (report.findings.some((finding) => finding.kind === "requires_reindex")) {
+        warnings.push("reindex requirement was not cleared; run 'familiar memory reindex --force'");
+    }
+    const summary = [`fixed ${fixed} item(s)`, ...warnings].join("; ");
+    return { fixed, summary };
+}
+function findDanglingIndexSources(stores, findings) {
+    const rows = stores.index.db
+        .prepare(`SELECT chunk_id, corpus, source_id, chunk_index
+			 FROM memory_index_sources
+			 WHERE chunk_id NOT IN (SELECT id FROM memory_chunks)
+			 ORDER BY corpus, source_id, chunk_index`)
+        .all();
+    for (const row of rows) {
+        findings.push({
+            kind: "dangling_index_source",
+            detail: `${row.corpus}:${row.source_id}#${row.chunk_index} references missing chunk ${row.chunk_id}`,
+            fixable: true,
+        });
+    }
+}
+function findOrphanEmptySegments(stores, findings) {
+    const rows = stores.lcm.db
+        .prepare(`SELECT id
+			 FROM lcm_segments
+			 WHERE status != 'active'
+			   AND id NOT IN (SELECT DISTINCT segment_id FROM lcm_records)
+			   AND id NOT IN (SELECT DISTINCT segment_id FROM lcm_summaries)
+			 ORDER BY started_at, id`)
+        .all();
+    for (const row of rows) {
+        findings.push({
+            kind: "orphan_empty_segment",
+            detail: `closed segment ${row.id} has no records`,
+            fixable: true,
+        });
+    }
+}
+function findStaleLcmIndexRows(stores, findings) {
+    const rows = stores.index.db
+        .prepare(`SELECT corpus, source_id, chunk_index
+			 FROM memory_index_sources
+			 WHERE corpus IN ('lcm_record', 'lcm_summary')
+			 ORDER BY corpus, source_id, chunk_index`)
+        .all();
+    for (const row of rows) {
+        if (lcmSourceExists(stores, row.corpus, row.source_id))
+            continue;
+        findings.push({
+            kind: "stale_lcm_index_source",
+            detail: `${row.corpus}:${row.source_id}#${row.chunk_index} points at missing LCM source`,
+            fixable: true,
+        });
+    }
+}
+function findBrokenContextOrdering(stores, findings) {
+    const sessions = stores.lcm.db
+        .prepare("SELECT DISTINCT session_key FROM lcm_context_items ORDER BY session_key")
+        .all();
+    for (const session of sessions) {
+        const rows = stores.lcm.db
+            .prepare(`SELECT ordinal
+				 FROM lcm_context_items
+				 WHERE session_key = ?
+				 ORDER BY ordinal`)
+            .all(session.session_key);
+        const ordinals = rows.map((row) => row.ordinal);
+        if (ordinals.every((ordinal, index) => ordinal === index) && new Set(ordinals).size === ordinals.length)
+            continue;
+        findings.push({
+            kind: "broken_context_ordering",
+            detail: `session ${session.session_key} ordinals are ${ordinals.join(",")}`,
+            fixable: true,
+        });
+    }
+}
+function findSummaryFkViolations(stores, findings) {
+    const sourceRows = stores.lcm.db
+        .prepare(`SELECT summary_id, ord, record_id
+			 FROM lcm_summary_sources
+			 WHERE record_id IS NOT NULL
+			   AND record_id NOT IN (SELECT id FROM lcm_records)
+			 ORDER BY summary_id, ord`)
+        .all();
+    for (const row of sourceRows) {
+        findings.push({
+            kind: "summary_fk_violation",
+            detail: `summary ${row.summary_id} source ${row.ord} references missing record ${row.record_id}`,
+            fixable: false,
+        });
+    }
+    const parentRows = stores.lcm.db
+        .prepare(`SELECT summary_id, parent_summary_id
+			 FROM lcm_summary_parents
+			 WHERE parent_summary_id NOT IN (SELECT id FROM lcm_summaries)
+			 ORDER BY summary_id, parent_summary_id`)
+        .all();
+    for (const row of parentRows) {
+        findings.push({
+            kind: "summary_fk_violation",
+            detail: `summary ${row.summary_id} references missing parent summary ${row.parent_summary_id}`,
+            fixable: false,
+        });
+    }
+}
+function findMissingPrunedSnapshots(stores, findings) {
+    const rows = stores.lcm.db
+        .prepare(`SELECT id
+			 FROM lcm_summaries
+			 WHERE covers_from_record_id IS NULL
+			   AND snapshot_json IS NULL
+			 ORDER BY id`)
+        .all();
+    for (const row of rows) {
+        findings.push({
+            kind: "missing_pruned_summary_snapshot",
+            detail: `summary ${row.id} has pruned raw coverage without snapshot_json`,
+            fixable: false,
+        });
+    }
+}
+function findRequiresReindex(stores, findings) {
+    const row = stores.index.db.prepare("SELECT v FROM memory_meta WHERE k = 'requires_reindex'").get();
+    if (row?.v !== "1")
+        return;
+    findings.push({
+        kind: "requires_reindex",
+        detail: "memory index was cleared after embedding config changed; run 'familiar memory reindex --force'",
+        fixable: false,
+    });
+}
+function findEmbeddingMismatches(stores, findings) {
+    const current = stores.index.embeddingConfig();
+    const rows = stores.index.db
+        .prepare(`SELECT id, corpus, embedding_model, embedding_dimensions
+			 FROM memory_chunks
+			 WHERE embedding_model != ? OR embedding_dimensions != ?
+			 ORDER BY id`)
+        .all(current.model, current.dimensions);
+    for (const row of rows) {
+        findings.push({
+            kind: "embedding_mismatch",
+            detail: `chunk ${row.id} (${row.corpus}) has ${row.embedding_model}/${row.embedding_dimensions}; ` +
+                `current is ${current.model}/${current.dimensions}`,
+            fixable: false,
+        });
+    }
+}
+function lcmSourceExists(stores, corpus, sourceId) {
+    if (corpus === "lcm_record") {
+        const id = parseIndexSourceId(sourceId, "lcm_record");
+        return id !== null && stores.lcm.getRecord(id) !== null;
+    }
+    if (corpus === "lcm_summary") {
+        const id = parseIndexSourceId(sourceId, "lcm_summary");
+        return id !== null && stores.lcm.getSummary(id) !== null;
+    }
+    return true;
+}
+function countIndexSourceRows(store, corpus, sourceId) {
+    const row = store.db
+        .prepare("SELECT COUNT(*) AS n FROM memory_index_sources WHERE corpus = ? AND source_id = ?")
+        .get(corpus, sourceId);
+    return row.n;
+}
+function parseIndexSourceId(value, prefix) {
+    const expectedPrefix = `${prefix}:`;
+    if (!value?.startsWith(expectedPrefix))
+        return null;
+    const id = Number(value.slice(expectedPrefix.length));
+    return Number.isInteger(id) && id > 0 ? id : null;
+}

package/dist/memory/index/chunk-indexer.js

@@ -0,0 +1,151 @@
+import { createMemoryContentHash } from "./store.js";
+export class ChunkIndexer {
+    store;
+    embeddingProvider;
+    constructor(options) {
+        this.store = options.store;
+        this.embeddingProvider = options.embeddingProvider;
+    }
+    async indexChunks(inputs, signal) {
+        const prepared = this.prepare(inputs);
+        if (prepared.length === 0)
+            return { ids: [], embedded: 0, reused: 0, skipped: inputs.length };
+        return this.insertPrepared(prepared, inputs.length - prepared.length, signal);
+    }
+    async replaceSource(corpus, sourceId, inputs, signal) {
+        const prepared = this.prepare(inputs.map((input) => ({ ...input, corpus, sourceId })));
+        const keepMappings = prepared.map((item) => ({ contentHash: item.contentHash, chunkIndex: item.chunkIndex }));
+        this.store.deleteBySourceExceptMappings(corpus, sourceId, keepMappings);
+        const result = await this.insertPrepared(prepared, inputs.length - prepared.length, signal);
+        return result;
+    }
+    prepare(inputs) {
+        const embeddingConfig = this.store.embeddingConfig();
+        const prepared = [];
+        for (const input of inputs) {
+            const text = input.text.trim();
+            if (!text)
+                continue;
+            const chunkIndex = input.chunkIndex ?? 0;
+            const sourceId = input.sourceId ?? null;
+            const contentHash = createMemoryContentHash({
+                corpus: input.corpus,
+                text,
+                embeddingModel: embeddingConfig.model,
+                embeddingDimensions: embeddingConfig.dimensions,
+            });
+            prepared.push({
+                input,
+                text,
+                chunkIndex,
+                sourceId,
+                contentHash,
+                existingId: null,
+                embedding: input.embedding,
+            });
+        }
+        return prepared;
+    }
+    async insertPrepared(prepared, skipped, signal) {
+        const startedAt = Date.now();
+        if (prepared.length === 0)
+            return { ids: [], embedded: 0, reused: 0, skipped };
+        const present = this.store.whichHashesPresent(prepared.map((item) => item.contentHash));
+        for (const item of prepared)
+            item.existingId = present.get(item.contentHash) ?? null;
+        const pendingEmbeddings = new Map();
+        const suppliedByHash = new Map();
+        for (const item of prepared) {
+            if (item.embedding) {
+                suppliedByHash.set(item.contentHash, item.embedding);
+                pendingEmbeddings.delete(item.contentHash);
+                continue;
+            }
+            if (item.existingId === null &&
+                !pendingEmbeddings.has(item.contentHash) &&
+                !suppliedByHash.has(item.contentHash)) {
+                pendingEmbeddings.set(item.contentHash, item);
+            }
+        }
+        const itemsToEmbed = [...pendingEmbeddings.values()];
+        let embeddingCost = 0;
+        const embeddings = itemsToEmbed.length === 0
+            ? []
+            : await this.embeddingProvider.embed(itemsToEmbed.map((item) => {
+                embeddingCost += item.text.length;
+                return item.text;
+            }), signal);
+        if (embeddings.length !== itemsToEmbed.length) {
+            throw new Error(`Embedding count mismatch: expected ${itemsToEmbed.length}, got ${embeddings.length}`);
+        }
+        for (let index = 0; index < itemsToEmbed.length; index++) {
+            const item = itemsToEmbed[index];
+            const embedding = embeddings[index];
+            if (!embedding)
+                throw new Error(`Embedding provider returned no result for chunk ${index}`);
+            item.embedding = embedding;
+        }
+        const embeddedByHash = new Map(suppliedByHash);
+        for (const item of itemsToEmbed)
+            embeddedByHash.set(item.contentHash, item.embedding);
+        const ids = new Array(prepared.length);
+        const toInsert = [];
+        const insertPositions = [];
+        const existingMappings = [];
+        for (let resultIndex = 0; resultIndex < prepared.length; resultIndex++) {
+            const item = prepared[resultIndex];
+            if (item.existingId !== null) {
+                ids[resultIndex] = item.existingId;
+                existingMappings.push({
+                    corpus: item.input.corpus,
+                    sourceId: item.sourceId,
+                    sourceRef: item.input.sourceRef ?? null,
+                    chunkIndex: item.chunkIndex,
+                    text: item.text,
+                    snippet: item.input.snippet,
+                    tokenCount: item.input.tokenCount ?? null,
+                    metadata: item.input.metadata ?? null,
+                    embedding: item.embedding ?? new Float32Array(this.store.embeddingConfig().dimensions),
+                });
+                continue;
+            }
+            const embedding = item.embedding ?? embeddedByHash.get(item.contentHash);
+            if (!embedding)
+                throw new Error("Missing embedding for memory chunk");
+            insertPositions.push(resultIndex);
+            toInsert.push({
+                corpus: item.input.corpus,
+                sourceId: item.sourceId,
+                sourceRef: item.input.sourceRef ?? null,
+                chunkIndex: item.chunkIndex,
+                text: item.text,
+                snippet: item.input.snippet,
+                tokenCount: item.input.tokenCount ?? null,
+                metadata: item.input.metadata ?? null,
+                embedding,
+            });
+        }
+        this.store.recordSourceMappings(existingMappings);
+        const insertedIds = this.store.insertChunks(toInsert);
+        for (let index = 0; index < insertPositions.length; index++) {
+            ids[insertPositions[index]] = insertedIds[index];
+        }
+        const result = {
+            ids,
+            embedded: itemsToEmbed.length,
+            reused: prepared.length - toInsert.length,
+            skipped,
+        };
+        logMemoryIndexBatch({
+            chunks: prepared.length,
+            durationMs: Date.now() - startedAt,
+            embeddingCost,
+        });
+        return result;
+    }
+}
+function logMemoryIndexBatch(payload) {
+    if (process.env.DEBUG !== "memory-index")
+        return;
+    console.error(JSON.stringify({ event: "memory_index_batch", ...payload }));
+}

package/dist/memory/index/embedding-provider.js

@@ -0,0 +1,119 @@
+export function createEmbeddingProvider(config, options = {}) {
+    const format = config.memory.embedding.format ?? config.memory.embedding.api;
+    if (format === "gemini")
+        return new GeminiEmbeddingProvider(config, options.fetchFn ?? fetch);
+    throw new Error(`NotImplementedError: memory.embedding.format=${format} is recognized but only gemini is implemented in v0`);
+}
+class GeminiEmbeddingProvider {
+    api = "gemini";
+    provider;
+    model;
+    dimensions;
+    baseUrl;
+    apiKeyEnv;
+    batchSize;
+    fetchFn;
+    constructor(config, fetchFn) {
+        this.provider = config.memory.embedding.provider;
+        this.model = config.memory.embedding.model;
+        this.dimensions = config.memory.embedding.dimensions;
+        this.baseUrl = config.memory.embedding.baseUrl.replace(/\/+$/, "");
+        this.apiKeyEnv = config.memory.embedding.apiKeyEnv;
+        this.batchSize = config.memory.embedding.batchSize;
+        if (this.batchSize < 1)
+            throw new Error(`Embedding batch size must be >= 1, got ${this.batchSize}`);
+        this.fetchFn = fetchFn;
+    }
+    async embedOne(input, signal) {
+        const [embedding] = await this.embed([input], signal);
+        if (!embedding)
+            throw new Error("Embedding provider returned no result");
+        return embedding;
+    }
+    async embed(inputs, signal) {
+        if (inputs.length === 0)
+            return [];
+        const embeddings = [];
+        for (let index = 0; index < inputs.length; index += this.batchSize) {
+            const chunk = inputs.slice(index, index + this.batchSize);
+            // Sequential batches are gentle on hosted rate limits; add bounded
+            // concurrency later if indexing throughput becomes a bottleneck.
+            embeddings.push(...(await this.embedBatch(chunk, signal)));
+        }
+        return embeddings;
+    }
+    async embedBatch(inputs, signal) {
+        const apiKey = this.apiKey();
+        const response = await this.fetchFn(this.buildUrl(), {
+            method: "POST",
+            headers: this.buildHeaders(apiKey),
+            body: JSON.stringify({
+                requests: inputs.map((input) => ({
+                    model: this.modelResourceName(),
+                    content: { parts: embeddingInputParts(input) },
+                    outputDimensionality: this.dimensions,
+                })),
+            }),
+            signal,
+        });
+        const { body, rawText } = await parseJsonResponse(response);
+        if (!response.ok) {
+            const message = typeof body.error?.message === "string"
+                ? body.error.message
+                : truncate(rawText.trim() || response.statusText);
+            throw new Error(`Embedding request failed: HTTP ${response.status} ${message}`.trim());
+        }
+        const rawEmbeddings = Array.isArray(body.embeddings) ? body.embeddings : body.embedding ? [body.embedding] : [];
+        if (rawEmbeddings.length !== inputs.length) {
+            throw new Error(`Embedding response count mismatch: expected ${inputs.length}, got ${rawEmbeddings.length}`);
+        }
+        return rawEmbeddings.map((embedding, index) => this.parseEmbeddingValues(embedding.values, index));
+    }
+    buildUrl() {
+        return `${this.baseUrl}/${this.modelResourceName()}:batchEmbedContents`;
+    }
+    buildHeaders(apiKey) {
+        const headers = { "content-type": "application/json" };
+        if (apiKey)
+            headers["x-goog-api-key"] = apiKey;
+        return headers;
+    }
+    apiKey() {
+        return this.apiKeyEnv ? process.env[this.apiKeyEnv] : undefined;
+    }
+    modelResourceName() {
+        return this.model.startsWith("models/") ? this.model : `models/${this.model}`;
+    }
+    parseEmbeddingValues(values, index) {
+        if (!Array.isArray(values) || values.some((value) => typeof value !== "number" || !Number.isFinite(value))) {
+            throw new Error(`Embedding response ${index} did not contain numeric values`);
+        }
+        if (values.length !== this.dimensions) {
+            throw new Error(`Embedding dimension mismatch for result ${index}: expected ${this.dimensions}, got ${values.length}`);
+        }
+        return new Float32Array(values);
+    }
+}
+function embeddingInputParts(input) {
+    if (typeof input === "string")
+        return [{ text: input }];
+    return input.parts.map((part) => {
+        if (part.type === "text")
+            return { text: part.text };
+        return { inlineData: { mimeType: part.mimeType, data: part.data } };
+    });
+}
+async function parseJsonResponse(response) {
+    const rawText = await response.text();
+    if (!rawText.trim())
+        return { body: {}, rawText };
+    try {
+        return { body: JSON.parse(rawText), rawText };
+    }
+    catch {
+        return { body: {}, rawText };
+    }
+}
+function truncate(text, maxLength = 300) {
+    return text.length <= maxLength ? text : `${text.slice(0, maxLength)}...`;
+}

package/dist/memory/index/fts-query.js

@@ -0,0 +1,18 @@
+export function normalizeFtsMatchQuery(query) {
+    const tokens = [];
+    for (const rawToken of query.normalize("NFKC").split(/\s+/u)) {
+        if (!rawToken)
+            continue;
+        const hasPrefix = rawToken.endsWith("*");
+        const body = hasPrefix ? rawToken.slice(0, -1) : rawToken;
+        const parts = body.match(/[\p{L}\p{N}_]+/gu) ?? [];
+        if (parts.length === 0)
+            continue;
+        for (let index = 0; index < parts.length; index++) {
+            const part = parts[index];
+            const suffix = hasPrefix && index === parts.length - 1 ? "*" : "";
+            tokens.push(`"${part.replaceAll('"', '""')}"${suffix}`);
+        }
+    }
+    return tokens.length > 0 ? tokens.join(" ") : null;
+}