@qearlyao/familiar 0.1.0

package/dist/memory/index/store.js

@@ -0,0 +1,513 @@
+import { createHash } from "node:crypto";
+import { mkdirSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import Database from "better-sqlite3";
+import { normalizeFtsMatchQuery } from "./fts-query.js";
+import { readMeta, runMemoryIndexMigrations } from "./schema.js";
+import { cosineDistance, decodeVector, encodeVector } from "./vector-codec.js";
+export class MemoryIndexStore {
+    db;
+    ownsDb;
+    embeddingProvider;
+    embeddingModel;
+    embeddingDimensions;
+    constructor(options) {
+        if (!options.db && !options.path)
+            throw new Error("MemoryIndexStore requires a db or path");
+        if (options.db) {
+            this.db = options.db;
+            this.ownsDb = false;
+        }
+        else {
+            const path = options.path;
+            mkdirSync(dirname(path), { recursive: true });
+            this.db = new Database(path);
+            this.ownsDb = true;
+        }
+        this.embeddingProvider = options.embeddingProvider;
+        this.embeddingModel = options.embeddingModel;
+        this.embeddingDimensions = options.embeddingDimensions;
+        runMemoryIndexMigrations(this.db, {
+            embeddingProvider: this.embeddingProvider,
+            embeddingModel: this.embeddingModel,
+            embeddingDimensions: this.embeddingDimensions,
+        });
+    }
+    static open(config) {
+        return new MemoryIndexStore({
+            path: resolve(config.memory.indexDir, "memory.sqlite"),
+            embeddingProvider: config.memory.embedding.provider,
+            embeddingModel: config.memory.embedding.model,
+            embeddingDimensions: config.memory.embedding.dimensions,
+        });
+    }
+    close() {
+        if (this.ownsDb)
+            this.db.close();
+    }
+    embeddingConfig() {
+        return {
+            provider: this.embeddingProvider,
+            model: this.embeddingModel,
+            dimensions: this.embeddingDimensions,
+        };
+    }
+    insertChunk(input) {
+        return this.insertChunks([input])[0];
+    }
+    insertChunks(inputs) {
+        if (inputs.length === 0)
+            return [];
+        const rows = inputs.map((input) => this.normalizeInput(input));
+        const out = [];
+        const insert = this.db.transaction((items) => {
+            for (const item of items)
+                out.push(this.insertNormalized(item));
+        });
+        insert.immediate(rows);
+        return out;
+    }
+    recordSourceMappings(inputs) {
+        if (inputs.length === 0)
+            return;
+        const rows = inputs.map((input) => this.normalizeInput(input));
+        this.db
+            .transaction((items) => {
+            for (const item of items) {
+                const existing = this.db
+                    .prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
+                    .get(item.contentHash);
+                if (existing)
+                    this.insertSourceMapping(existing.id, item);
+            }
+        })
+            .immediate(rows);
+    }
+    replaceSource(corpus, sourceId, inputs) {
+        const rows = inputs.map((input) => this.normalizeInput({ ...input, corpus, sourceId }));
+        const out = [];
+        const replace = this.db.transaction(() => {
+            this.deleteBySourceInternal(corpus, sourceId);
+            for (const item of rows)
+                out.push(this.insertNormalized(item));
+        });
+        replace.immediate();
+        return out;
+    }
+    whichHashesPresent(hashes) {
+        const present = new Map();
+        if (hashes.length === 0)
+            return present;
+        const chunkSize = 256;
+        for (let index = 0; index < hashes.length; index += chunkSize) {
+            const chunk = hashes.slice(index, index + chunkSize);
+            const placeholders = chunk.map(() => "?").join(",");
+            const rows = this.db
+                .prepare(`SELECT content_hash, id FROM memory_chunks WHERE content_hash IN (${placeholders})`)
+                .all(...chunk);
+            for (const row of rows)
+                present.set(row.content_hash, row.id);
+        }
+        return present;
+    }
+    getChunk(id) {
+        const row = this.db
+            .prepare(`SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.id = ?`)
+            .get(id);
+        return row ? rowToChunk(row) : null;
+    }
+    searchLexical(query, options = {}) {
+        const normalized = normalizeSearchOptions(options);
+        const matchQuery = normalizeFtsMatchQuery(query);
+        if (!matchQuery)
+            return [];
+        const params = [matchQuery];
+        const corpusFilter = normalized.corpus ? "AND c.corpus = ?" : "";
+        if (normalized.corpus)
+            params.push(normalized.corpus);
+        params.push(normalized.limit);
+        const rows = this.db
+            .prepare(`SELECT c.*, f.rank AS score, ${sourcesJsonSelect("c.id")}
+				 FROM memory_fts f
+				 JOIN memory_chunks c ON c.id = f.rowid
+				 WHERE memory_fts MATCH ?
+				 ${corpusFilter}
+				 ORDER BY f.rank
+				 LIMIT ?`)
+            .all(...params);
+        return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
+    }
+    searchSemantic(query, options = {}) {
+        const normalized = normalizeSearchOptions(options);
+        if (query.length !== this.embeddingDimensions) {
+            throw new Error(`Query vector dimension mismatch: expected ${this.embeddingDimensions}, got ${query.length}`);
+        }
+        // memory_vec does not carry corpus metadata, so sqlite-vec cannot prefilter
+        // corpus-scoped KNN. Use the linear path to keep scoped nearest neighbors exact.
+        if (this.vectorCapability() === "sqlite-vec" && !normalized.corpus)
+            return this.searchSemanticVec(query, normalized);
+        return this.searchSemanticLinear(query, normalized);
+    }
+    searchSemanticVec(query, normalized) {
+        const params = [encodeVector(query), normalized.limit];
+        if (normalized.corpus)
+            params.push(normalized.corpus);
+        const corpusFilter = normalized.corpus ? "WHERE c.corpus = ?" : "";
+        const rows = this.db
+            .prepare(`SELECT c.*, v.distance AS score, ${sourcesJsonSelect("c.id")}
+				 FROM (
+					SELECT rowid AS chunk_id, distance
+					FROM memory_vec
+					WHERE embedding MATCH ? AND k = ?
+				 ) v
+				 JOIN memory_chunks c ON c.id = v.chunk_id
+				 ${corpusFilter}
+				 ORDER BY v.distance
+				 LIMIT ?`)
+            .all(...params, normalized.limit);
+        return rows.map((row) => ({ id: row.id, score: row.score, chunk: rowToChunk(row) }));
+    }
+    searchSemanticLinear(query, normalized) {
+        const rows = this.db
+            .prepare(normalized.corpus
+            ? `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c WHERE c.corpus = ?`
+            : `SELECT c.*, ${sourcesJsonSelect("c.id")} FROM memory_chunks c`)
+            .all(...(normalized.corpus ? [normalized.corpus] : []));
+        return rows
+            .map((row) => ({
+            id: row.id,
+            score: cosineDistance(query, decodeVector(row.embedding, row.embedding_dimensions)),
+            chunk: rowToChunk(row),
+        }))
+            .sort((a, b) => a.score - b.score)
+            .slice(0, normalized.limit);
+    }
+    deleteChunk(id) {
+        const remove = this.db.transaction(() => {
+            this.deleteFtsRow(id);
+            this.db.prepare("DELETE FROM memory_chunks WHERE id = ?").run(id);
+        });
+        remove.immediate();
+    }
+    deleteBySource(corpus, sourceId) {
+        this.db.transaction(() => this.deleteBySourceInternal(corpus, sourceId)).immediate();
+    }
+    /** Caller already owns the index DB write transaction. */
+    deleteBySourceUnsafe(corpus, sourceId) {
+        this.deleteBySourceInternal(corpus, sourceId);
+    }
+    getSourceState(corpus, sourceId) {
+        const row = this.db
+            .prepare(`SELECT
+					st.corpus,
+					st.source_id,
+					st.source_ref,
+					st.mtime_ms,
+					st.size_bytes,
+					st.updated_at,
+					EXISTS(
+						SELECT 1 FROM memory_index_sources s
+						WHERE s.corpus = st.corpus AND s.source_id = st.source_id
+					) AS has_mappings
+				 FROM memory_index_source_state st
+				 WHERE st.corpus = ? AND st.source_id = ?`)
+            .get(corpus, sourceId);
+        return row
+            ? {
+                corpus: row.corpus,
+                sourceId: row.source_id,
+                sourceRef: row.source_ref,
+                mtimeMs: row.mtime_ms,
+                sizeBytes: row.size_bytes,
+                updatedAt: row.updated_at,
+                hasMappings: row.has_mappings === 1,
+            }
+            : null;
+    }
+    upsertSourceState(input) {
+        this.db
+            .prepare(`INSERT INTO memory_index_source_state(corpus, source_id, source_ref, mtime_ms, size_bytes)
+				 VALUES (?, ?, ?, ?, ?)
+				 ON CONFLICT(corpus, source_id) DO UPDATE SET
+					source_ref = excluded.source_ref,
+					mtime_ms = excluded.mtime_ms,
+					size_bytes = excluded.size_bytes,
+					updated_at = unixepoch()`)
+            .run(input.corpus, input.sourceId, input.sourceRef ?? null, Math.floor(input.mtimeMs), input.sizeBytes);
+    }
+    deleteBySourceExceptHashes(corpus, sourceId, contentHashes) {
+        this.deleteBySourceExceptMappings(corpus, sourceId, [...new Set(contentHashes)].map((contentHash) => ({ contentHash, chunkIndex: null })));
+    }
+    deleteBySourceExceptMappings(corpus, sourceId, kept) {
+        if (kept.length === 0) {
+            this.deleteBySource(corpus, sourceId);
+            return;
+        }
+        this.db
+            .transaction(() => {
+            const rows = this.db
+                .prepare(`SELECT s.chunk_id AS id, c.content_hash, s.chunk_index
+						 FROM memory_index_sources s
+						 JOIN memory_chunks c ON c.id = s.chunk_id
+						 WHERE s.corpus = ? AND s.source_id = ?`)
+                .all(corpus, sourceId);
+            for (const row of rows) {
+                if (kept.some((item) => item.contentHash === row.content_hash &&
+                    (item.chunkIndex === null || item.chunkIndex === row.chunk_index))) {
+                    continue;
+                }
+                this.db
+                    .prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
+                    .run(corpus, sourceId, row.chunk_index);
+                this.deleteOrphanChunk(row.id);
+            }
+        })
+            .immediate();
+    }
+    clearAll() {
+        this.db
+            .transaction(() => {
+            this.db.prepare("INSERT INTO memory_fts(memory_fts) VALUES ('delete-all')").run();
+            if (this.vectorCapability() === "sqlite-vec")
+                this.db.prepare("DELETE FROM memory_vec").run();
+            this.db.prepare("DELETE FROM memory_index_sources").run();
+            this.db.prepare("DELETE FROM memory_index_source_state").run();
+            this.db.prepare("DELETE FROM memory_chunks").run();
+        })
+            .immediate();
+    }
+    reconcileSources(exists) {
+        const sources = this.db
+            .prepare("SELECT chunk_id, corpus, source_id, source_ref, chunk_index FROM memory_index_sources")
+            .all();
+        if (sources.length === 0)
+            return;
+        this.db
+            .transaction(() => {
+            for (const row of sources) {
+                const source = {
+                    corpus: row.corpus,
+                    sourceId: row.source_id,
+                    sourceRef: row.source_ref,
+                    chunkIndex: row.chunk_index,
+                };
+                if (exists(source))
+                    continue;
+                this.db
+                    .prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ? AND chunk_index = ?")
+                    .run(source.corpus, source.sourceId, source.chunkIndex);
+                this.deleteOrphanChunk(row.chunk_id);
+            }
+        })
+            .immediate();
+    }
+    stats() {
+        const indexed = this.db.prepare("SELECT COUNT(*) AS n FROM memory_chunks").get().n;
+        const ftsRows = this.db.prepare("SELECT COUNT(*) AS n FROM memory_fts").get().n;
+        const size = this.db
+            .prepare("SELECT page_count * page_size AS bytes FROM pragma_page_count(), pragma_page_size()")
+            .get();
+        const dimensionsRaw = readMeta(this.db, "embedding_dimensions");
+        const vectorCapability = this.vectorCapability();
+        const vectorRows = vectorCapability === "sqlite-vec" ? this.vectorRowCount() : indexed;
+        return {
+            indexed,
+            ftsRows,
+            vectorRows,
+            vectorAvailable: vectorCapability === "sqlite-vec",
+            vectorCapability,
+            requiresReindex: readMeta(this.db, "requires_reindex") === "1",
+            embeddingProvider: readMeta(this.db, "embedding_provider"),
+            embeddingModel: readMeta(this.db, "embedding_model"),
+            embeddingDimensions: dimensionsRaw ? Number(dimensionsRaw) : null,
+            dbSizeBytes: size.bytes,
+        };
+    }
+    vectorCapability() {
+        return readMeta(this.db, "vector_capability") === "sqlite-vec" ? "sqlite-vec" : "blob-js";
+    }
+    vectorRowCount() {
+        try {
+            const row = this.db.prepare("SELECT COUNT(*) AS n FROM memory_vec").get();
+            return row.n;
+        }
+        catch {
+            return 0;
+        }
+    }
+    normalizeInput(input) {
+        if (input.embedding.length !== this.embeddingDimensions) {
+            throw new Error(`Embedding dimension mismatch: expected ${this.embeddingDimensions}, got ${input.embedding.length}`);
+        }
+        const text = input.text.trim();
+        if (!text)
+            throw new Error("Memory chunk text must not be empty");
+        const chunkIndex = input.chunkIndex ?? 0;
+        const snippet = input.snippet?.trim() || text.slice(0, 280);
+        return {
+            corpus: input.corpus,
+            sourceId: input.sourceId ?? null,
+            sourceRef: input.sourceRef ?? null,
+            chunkIndex,
+            text,
+            snippet,
+            tokenCount: input.tokenCount ?? null,
+            metadataJson: input.metadata ? JSON.stringify(input.metadata) : null,
+            embedding: input.embedding,
+            contentHash: createMemoryContentHash({
+                corpus: input.corpus,
+                text,
+                embeddingModel: this.embeddingModel,
+                embeddingDimensions: this.embeddingDimensions,
+            }),
+        };
+    }
+    insertNormalized(item) {
+        const existing = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
+        if (existing) {
+            this.insertSourceMapping(existing.id, item);
+            return existing.id;
+        }
+        const result = this.db
+            .prepare(`INSERT INTO memory_chunks (
+					content_hash, corpus, text_full, snippet, token_count, metadata_json, embedding_model,
+					embedding_dimensions, embedding
+				 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+            .run(item.contentHash, item.corpus, item.text, item.snippet, item.tokenCount, item.metadataJson, this.embeddingModel, this.embeddingDimensions, encodeVector(item.embedding));
+        const id = Number(result.lastInsertRowid);
+        this.db
+            .prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)")
+            .run(id, item.text, item.snippet);
+        if (this.vectorCapability() === "sqlite-vec") {
+            this.db
+                .prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
+                .run(id, encodeVector(item.embedding));
+        }
+        this.insertSourceMapping(id, item);
+        return id;
+    }
+    deleteBySourceInternal(corpus, sourceId) {
+        const rows = this.db
+            .prepare("SELECT chunk_id AS id FROM memory_index_sources WHERE corpus = ? AND source_id = ?")
+            .all(corpus, sourceId);
+        this.db.prepare("DELETE FROM memory_index_sources WHERE corpus = ? AND source_id = ?").run(corpus, sourceId);
+        this.db.prepare("DELETE FROM memory_index_source_state WHERE corpus = ? AND source_id = ?").run(corpus, sourceId);
+        for (const row of rows)
+            this.deleteOrphanChunk(row.id);
+    }
+    insertSourceMapping(chunkId, item) {
+        if (!item.sourceId)
+            return;
+        this.db
+            .prepare(`INSERT OR REPLACE INTO memory_index_sources(chunk_id, corpus, source_id, source_ref, chunk_index)
+				 VALUES (?, ?, ?, ?, ?)`)
+            .run(chunkId, item.corpus, item.sourceId, item.sourceRef, item.chunkIndex);
+    }
+    deleteOrphanChunk(id) {
+        const remaining = this.db
+            .prepare("SELECT 1 AS ok FROM memory_index_sources WHERE chunk_id = ? LIMIT 1")
+            .get(id);
+        if (remaining)
+            return;
+        this.deleteFtsRow(id);
+        this.db.prepare("DELETE FROM memory_chunks WHERE id = ?").run(id);
+    }
+    deleteFtsRow(id) {
+        this.db.prepare("DELETE FROM memory_fts WHERE rowid = ?").run(id);
+    }
+}
+export function createMemoryContentHash(input) {
+    return createHash("sha256")
+        .update(JSON.stringify({
+        corpus: input.corpus,
+        text: input.text,
+        embeddingModel: input.embeddingModel,
+        embeddingDimensions: input.embeddingDimensions,
+    }))
+        .digest("hex");
+}
+function normalizeSearchOptions(options) {
+    if (typeof options === "number")
+        return { limit: options };
+    return {
+        limit: options.limit ?? 10,
+        corpus: options.corpus,
+    };
+}
+function sourcesJsonSelect(chunkIdExpr) {
+    return `(SELECT json_group_array(json_object(
+		'corpus', s.corpus,
+		'sourceId', s.source_id,
+		'sourceRef', s.source_ref,
+		'chunkIndex', s.chunk_index
+	)) FROM memory_index_sources s WHERE s.chunk_id = ${chunkIdExpr}) AS sources_json`;
+}
+function rowToChunk(row) {
+    const sources = sourceRefsFromRow(row);
+    const primary = sources[0] ?? {
+        corpus: row.corpus,
+        sourceId: row.source_id ?? null,
+        sourceRef: row.source_ref ?? null,
+        chunkIndex: row.chunk_index ?? 0,
+    };
+    return {
+        id: row.id,
+        contentHash: row.content_hash,
+        corpus: row.corpus,
+        sourceId: primary.sourceId,
+        sourceRef: primary.sourceRef,
+        chunkIndex: primary.chunkIndex,
+        sources,
+        text: row.text_full,
+        snippet: row.snippet,
+        tokenCount: row.token_count,
+        metadata: parseMetadata(row.metadata_json),
+        embeddingModel: row.embedding_model,
+        embeddingDimensions: row.embedding_dimensions,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+function sourceRefsFromRow(row) {
+    if ("sources_json" in row && typeof row.sources_json === "string" && row.sources_json) {
+        try {
+            const parsed = JSON.parse(row.sources_json);
+            if (Array.isArray(parsed)) {
+                return parsed.filter(isSourceRef);
+            }
+        }
+        catch {
+            return [];
+        }
+    }
+    const sourceId = row.source_id ?? null;
+    return sourceId
+        ? [
+            {
+                corpus: row.corpus,
+                sourceId,
+                sourceRef: row.source_ref ?? null,
+                chunkIndex: row.chunk_index ?? 0,
+            },
+        ]
+        : [];
+}
+function isSourceRef(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const item = value;
+    return typeof item.corpus === "string" && typeof item.sourceId === "string";
+}
+function parseMetadata(value) {
+    if (!value)
+        return null;
+    try {
+        const parsed = JSON.parse(value);
+        return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/memory/index/vec.js

@@ -0,0 +1,72 @@
+import { createRequire } from "node:module";
+const requireOptional = createRequire(import.meta.url);
+let loggedUnavailable = false;
+let loadModule = () => requireOptional("sqlite-vec");
+let loadedDbs = new WeakSet();
+export function loadSqliteVec(db) {
+    if (loadedDbs.has(db))
+        return availableState;
+    try {
+        const mod = loadModule();
+        registerModuleOnDb(mod, db);
+        const row = db.prepare("SELECT vec_version() AS version").get();
+        if (!row?.version)
+            throw new Error("sqlite-vec loaded but vec_version() is unavailable");
+        loadedDbs.add(db);
+        return availableState;
+    }
+    catch {
+        logUnavailableOnce();
+        return { available: false };
+    }
+}
+export function isSqliteVecLoadedForDb(db) {
+    return loadedDbs.has(db);
+}
+function registerModuleOnDb(mod, db) {
+    if (typeof mod.load === "function") {
+        mod.load(db);
+        return;
+    }
+    if (typeof mod.default?.load === "function") {
+        mod.default.load(db);
+        return;
+    }
+    if (typeof mod.loadablePathFor === "function") {
+        const loadExtension = db.loadExtension;
+        if (typeof loadExtension !== "function")
+            throw new Error("better-sqlite3 does not support loadExtension");
+        loadExtension.call(db, mod.loadablePathFor());
+        return;
+    }
+    throw new Error("sqlite-vec module does not expose load(db) or loadablePathFor()");
+}
+function logUnavailableOnce() {
+    if (loggedUnavailable)
+        return;
+    loggedUnavailable = true;
+    console.info("sqlite-vec module unavailable; using JS linear scan over stored embeddings");
+}
+const availableState = {
+    available: true,
+    registerOnDb(db) {
+        const state = loadSqliteVec(db);
+        if (!state.available)
+            throw new Error("sqlite-vec unavailable");
+    },
+};
+export const __memoryVecTest = {
+    setLoader(loader) {
+        loadModule = loader ?? (() => requireOptional("sqlite-vec"));
+        loadedDbs = new WeakSet();
+        loggedUnavailable = false;
+    },
+    probePackage() {
+        try {
+            return requireOptional("sqlite-vec");
+        }
+        catch {
+            return null;
+        }
+    },
+};

package/dist/memory/index/vector-codec.js

@@ -0,0 +1,27 @@
+export function encodeVector(vector) {
+    return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+export function decodeVector(buffer, dimensions) {
+    if (buffer.byteLength !== dimensions * Float32Array.BYTES_PER_ELEMENT) {
+        throw new Error(`Vector blob dimension mismatch: expected ${dimensions}, got ${buffer.byteLength / 4}`);
+    }
+    const bytes = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+    return new Float32Array(bytes);
+}
+export function cosineDistance(a, b) {
+    if (a.length !== b.length)
+        throw new Error(`Vector dimension mismatch: ${a.length} !== ${b.length}`);
+    let dot = 0;
+    let aNorm = 0;
+    let bNorm = 0;
+    for (let index = 0; index < a.length; index++) {
+        const av = a[index] ?? 0;
+        const bv = b[index] ?? 0;
+        dot += av * bv;
+        aNorm += av * av;
+        bNorm += bv * bv;
+    }
+    if (aNorm === 0 || bNorm === 0)
+        return 1;
+    return 1 - dot / (Math.sqrt(aNorm) * Math.sqrt(bNorm));
+}