@qearlyao/familiar 0.1.0

package/dist/memory/index/retrieval.js

@@ -0,0 +1,246 @@
+const DEFAULT_LIMIT = 8;
+const DEFAULT_CANDIDATE_MULTIPLIER = 4;
+const RRF_K = 60;
+const TEXT_DEDUPE_MIN_CHARS = 24;
+export async function retrieveMemory(options) {
+    const query = options.query.trim();
+    if (!query)
+        return [];
+    const limit = positiveIntegerOrDefault(options.limit, DEFAULT_LIMIT);
+    const candidateLimit = positiveIntegerOrDefault(options.candidateLimit, Math.max(limit * DEFAULT_CANDIDATE_MULTIPLIER, limit));
+    const useLexical = options.useLexical ?? true;
+    const useSemantic = options.useSemantic ?? Boolean(options.embeddingProvider);
+    const lexicalHits = useLexical ? searchLexicalByScope(options.store, query, candidateLimit, options.scope) : [];
+    const semanticHits = useSemantic && options.embeddingProvider
+        ? await searchSemanticByScope(options.store, options.embeddingProvider, query, candidateLimit, options.scope, options.signal)
+        : [];
+    return dedupeMemoryHits(mergeRankedHits(lexicalHits, semanticHits, options.scope)).slice(0, limit);
+}
+function searchLexicalByScope(store, query, limit, scope) {
+    const corpora = uniqueStrings(scope?.corpora);
+    if (corpora.length === 0)
+        return store.searchLexical(query, { limit });
+    return corpora.flatMap((corpus) => store.searchLexical(query, { limit, corpus }));
+}
+async function searchSemanticByScope(store, provider, query, limit, scope, signal) {
+    let vector;
+    try {
+        vector = await provider.embedOne(query, signal);
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        return [];
+    }
+    const corpora = uniqueStrings(scope?.corpora);
+    if (corpora.length === 0)
+        return store.searchSemantic(vector, { limit });
+    return corpora.flatMap((corpus) => store.searchSemantic(vector, { limit, corpus }));
+}
+function mergeRankedHits(lexicalHits, semanticHits, scope) {
+    const merged = new Map();
+    addHits(merged, lexicalHits, "lexical", scope);
+    addHits(merged, semanticHits, "semantic", scope);
+    return Array.from(merged.values()).sort(compareRetrievalHits);
+}
+function addHits(merged, hits, channel, scope) {
+    const ranksByCorpus = new Map();
+    const rankByCorpus = uniqueStrings(scope?.corpora).length > 0;
+    for (const hit of hits) {
+        if (!matchesScope(hit.chunk, scope))
+            continue;
+        const corpus = rankByCorpus ? hit.chunk.corpus : "";
+        // Corpus-scoped searches are independent retriever lists; each corpus starts
+        // RRF rank at 1 so fan-out order does not penalize later corpora.
+        const rank = (ranksByCorpus.get(corpus) ?? 0) + 1;
+        ranksByCorpus.set(corpus, rank);
+        const existing = merged.get(hit.id);
+        if (!existing) {
+            merged.set(hit.id, {
+                id: hit.id,
+                score: reciprocalRank(rank),
+                chunk: hit.chunk,
+                lexicalRank: channel === "lexical" ? rank : null,
+                semanticRank: channel === "semantic" ? rank : null,
+                lexicalScore: channel === "lexical" ? hit.score : null,
+                semanticScore: channel === "semantic" ? hit.score : null,
+            });
+            continue;
+        }
+        existing.score += reciprocalRank(rank);
+        if (channel === "lexical") {
+            existing.lexicalRank = rank;
+            existing.lexicalScore = hit.score;
+        }
+        else {
+            existing.semanticRank = rank;
+            existing.semanticScore = hit.score;
+        }
+    }
+}
+function compareRetrievalHits(a, b) {
+    return (b.score - a.score ||
+        bestRank(a) - bestRank(b) ||
+        (a.semanticScore ?? Number.POSITIVE_INFINITY) - (b.semanticScore ?? Number.POSITIVE_INFINITY) ||
+        (a.lexicalScore ?? Number.POSITIVE_INFINITY) - (b.lexicalScore ?? Number.POSITIVE_INFINITY) ||
+        a.id - b.id);
+}
+function bestRank(hit) {
+    return Math.min(hit.lexicalRank ?? Number.POSITIVE_INFINITY, hit.semanticRank ?? Number.POSITIVE_INFINITY);
+}
+function dedupeMemoryHits(hits) {
+    const groups = [];
+    const groupByKey = new Map();
+    for (const hit of hits) {
+        const keys = memoryDedupeKeys(hit.chunk);
+        const groupIndexes = new Set();
+        for (const key of keys) {
+            const groupIndex = groupByKey.get(key);
+            if (groupIndex !== undefined)
+                groupIndexes.add(groupIndex);
+        }
+        const targetIndex = groupIndexes.size > 0 ? Math.min(...groupIndexes) : groups.length;
+        const target = groups[targetIndex] ?? [];
+        target.push(hit);
+        groups[targetIndex] = target;
+        for (const groupIndex of groupIndexes) {
+            if (groupIndex === targetIndex)
+                continue;
+            for (const grouped of groups[groupIndex] ?? []) {
+                target.push(grouped);
+                for (const key of memoryDedupeKeys(grouped.chunk))
+                    groupByKey.set(key, targetIndex);
+            }
+            groups[groupIndex] = [];
+        }
+        for (const key of keys)
+            groupByKey.set(key, targetIndex);
+    }
+    return groups
+        .filter((group) => group.length > 0)
+        .map((group) => group.sort(compareRetrievalHits)[0])
+        .filter((hit) => hit !== undefined)
+        .sort(compareRetrievalHits);
+}
+function memoryDedupeKeys(chunk) {
+    const keys = new Set();
+    const text = normalizeMemoryText(chunk.text);
+    if (text) {
+        if (text.length >= TEXT_DEDUPE_MIN_CHARS)
+            keys.add(`text:${chunk.corpus}:${text}`);
+        const kind = metadataString(chunk.metadata, "kind") ?? "";
+        const rounded = roundedChunkTimestamp(chunk);
+        if (kind && rounded !== null)
+            keys.add(`turn:${chunk.corpus}:${kind}:${rounded}:${text}`);
+    }
+    const sourceMessageId = metadataString(chunk.metadata, "sourceMessageId") ?? metadataSourceString(chunk, "sourceMessageId");
+    if (sourceMessageId)
+        keys.add(`message:${chunk.corpus}:${sourceMessageId}`);
+    return [...keys];
+}
+function normalizeMemoryText(text) {
+    const normalized = text
+        .replace(/^\s*\[[^\]]+\]\s*/, "")
+        .replace(/\s+/g, " ")
+        .trim()
+        .toLowerCase();
+    // Transitional shim for already-indexed legacy chunks that duplicated visible text.
+    const half = normalized.length / 2;
+    if (Number.isInteger(half) && normalized.slice(0, half).trim() === normalized.slice(half).trim()) {
+        return normalized.slice(0, half).trim();
+    }
+    return normalized;
+}
+function roundedChunkTimestamp(chunk) {
+    const timestamp = chunkTimestamp(chunk);
+    return timestamp === null ? null : Math.round(timestamp / 60_000);
+}
+function metadataString(metadata, key) {
+    const value = metadata?.[key];
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+function metadataSourceString(chunk, key) {
+    const source = chunk.metadata?.source;
+    if (!source || typeof source !== "object")
+        return null;
+    const value = source[key];
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+function reciprocalRank(rank) {
+    return 1 / (RRF_K + rank);
+}
+function matchesScope(chunk, scope) {
+    const corpora = uniqueStrings(scope?.corpora);
+    if (corpora.length > 0 && !corpora.includes(chunk.corpus))
+        return false;
+    if (!matchesTimeScope(chunk, scope))
+        return false;
+    const sourceIds = uniqueStrings(scope?.sourceIds);
+    if (sourceIds.length > 0 &&
+        !chunkSources(chunk).some((source) => source.sourceId && sourceIds.includes(source.sourceId))) {
+        return false;
+    }
+    const sourceRefs = uniqueStrings(scope?.sourceRefs);
+    if (sourceRefs.length > 0 &&
+        !chunkSources(chunk).some((source) => source.sourceRef && sourceRefs.includes(source.sourceRef))) {
+        return false;
+    }
+    return true;
+}
+function matchesTimeScope(chunk, scope) {
+    const after = parseIsoTime(scope?.after);
+    const before = parseIsoTime(scope?.before);
+    if (after === null && before === null)
+        return true;
+    const timestamp = chunkTimestamp(chunk);
+    if (timestamp === null)
+        return false;
+    if (after !== null && timestamp < after)
+        return false;
+    if (before !== null && timestamp > before)
+        return false;
+    return true;
+}
+function chunkTimestamp(chunk) {
+    const raw = firstMetadataValue(chunk.metadata, [
+        "timestamp",
+        "happenedAt",
+        "coverageToHappenedAt",
+        "coverageFromHappenedAt",
+    ]);
+    if (typeof raw === "string") {
+        const parsed = Date.parse(raw);
+        if (Number.isFinite(parsed))
+            return parsed;
+    }
+    if (typeof raw === "number" && Number.isFinite(raw))
+        return raw < 10_000_000_000 ? raw * 1000 : raw;
+    return chunk.createdAt < 10_000_000_000 ? chunk.createdAt * 1000 : chunk.createdAt;
+}
+function firstMetadataValue(metadata, keys) {
+    if (!metadata)
+        return null;
+    for (const key of keys) {
+        const value = metadata[key];
+        if (typeof value === "string" || typeof value === "number")
+            return value;
+    }
+    return null;
+}
+function parseIsoTime(value) {
+    if (!value)
+        return null;
+    const parsed = Date.parse(value);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function chunkSources(chunk) {
+    return chunk.sources.length > 0 || !chunk.sourceId
+        ? chunk.sources
+        : [{ corpus: chunk.corpus, sourceId: chunk.sourceId, sourceRef: chunk.sourceRef, chunkIndex: chunk.chunkIndex }];
+}
+function uniqueStrings(values) {
+    return Array.from(new Set(values?.filter((value) => value.trim()) ?? []));
+}
+function positiveIntegerOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
+}

package/dist/memory/index/schema.js

@@ -0,0 +1,157 @@
+import { loadSqliteVec } from "./vec.js";
+const SCHEMA_VERSION = 4;
+export function runMemoryIndexMigrations(db, options) {
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    const vec = loadSqliteVec(db);
+    db.exec(`
+		CREATE TABLE IF NOT EXISTS memory_meta (
+			k TEXT PRIMARY KEY,
+			v TEXT NOT NULL
+		);
+
+		CREATE TABLE IF NOT EXISTS memory_chunks (
+			id INTEGER PRIMARY KEY,
+			content_hash TEXT NOT NULL UNIQUE,
+			corpus TEXT NOT NULL,
+			text_full TEXT NOT NULL,
+			snippet TEXT NOT NULL,
+			token_count INTEGER,
+			metadata_json TEXT,
+			embedding_model TEXT NOT NULL,
+			embedding_dimensions INTEGER NOT NULL,
+			embedding BLOB NOT NULL,
+			created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+			updated_at INTEGER NOT NULL DEFAULT (unixepoch())
+		);
+
+		CREATE INDEX IF NOT EXISTS idx_memory_chunks_hash ON memory_chunks(content_hash);
+		CREATE INDEX IF NOT EXISTS idx_memory_chunks_model ON memory_chunks(embedding_model, embedding_dimensions);
+
+		CREATE TABLE IF NOT EXISTS memory_index_sources (
+			chunk_id INTEGER NOT NULL REFERENCES memory_chunks(id) ON DELETE CASCADE,
+			corpus TEXT NOT NULL,
+			source_id TEXT NOT NULL,
+			source_ref TEXT,
+			chunk_index INTEGER NOT NULL DEFAULT 0,
+			PRIMARY KEY(corpus, source_id, chunk_index)
+		);
+
+		CREATE INDEX IF NOT EXISTS idx_memory_index_sources_chunk ON memory_index_sources(chunk_id);
+
+		CREATE TABLE IF NOT EXISTS memory_index_source_state (
+			corpus TEXT NOT NULL,
+			source_id TEXT NOT NULL,
+			source_ref TEXT,
+			mtime_ms INTEGER NOT NULL,
+			size_bytes INTEGER NOT NULL,
+			updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
+			PRIMARY KEY(corpus, source_id)
+		);
+
+		-- Contentless FTS avoids SQLite maintaining shadow copies or stale external-content rows.
+		CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+			text_full,
+			snippet,
+			content='',
+			contentless_delete=1
+		);
+	`);
+    migrateMemoryIndexSources(db);
+    reconcileEmbeddingConfig(db, options);
+    const vectorCapability = reconcileVectorTable(db, options, vec.available);
+    writeMeta(db, "schema_version", String(SCHEMA_VERSION));
+    writeMeta(db, "embedding_provider", options.embeddingProvider);
+    writeMeta(db, "embedding_model", options.embeddingModel);
+    writeMeta(db, "embedding_dimensions", String(options.embeddingDimensions));
+    writeMeta(db, "vector_capability", vectorCapability);
+}
+export function readMeta(db, key) {
+    const row = db.prepare("SELECT v FROM memory_meta WHERE k = ?").get(key);
+    return row?.v ?? null;
+}
+export function writeMeta(db, key, value) {
+    db.prepare(`INSERT INTO memory_meta(k, v) VALUES (?, ?)
+		 ON CONFLICT(k) DO UPDATE SET v = excluded.v`).run(key, value);
+}
+function reconcileEmbeddingConfig(db, options) {
+    const model = readMeta(db, "embedding_model");
+    const dimensions = readMeta(db, "embedding_dimensions");
+    if ((model && model !== options.embeddingModel) ||
+        (dimensions && dimensions !== String(options.embeddingDimensions))) {
+        db.transaction(() => {
+            db.prepare("DELETE FROM memory_fts").run();
+            db.prepare("DROP TRIGGER IF EXISTS trg_memory_chunks_delete_vec").run();
+            db.prepare("DROP TABLE IF EXISTS memory_vec").run();
+            db.prepare("DELETE FROM memory_index_source_state").run();
+            db.prepare("DELETE FROM memory_index_sources").run();
+            db.prepare("DELETE FROM memory_chunks").run();
+            writeMeta(db, "requires_reindex", "1");
+        }).immediate();
+    }
+}
+function reconcileVectorTable(db, options, sqliteVecAvailable) {
+    const previousCapability = readMeta(db, "vector_capability");
+    if (!sqliteVecAvailable) {
+        db.prepare("DROP TRIGGER IF EXISTS trg_memory_chunks_delete_vec").run();
+        return "blob-js";
+    }
+    try {
+        db.transaction(() => {
+            const hadVectorTable = tableExists(db, "memory_vec");
+            if (!hadVectorTable) {
+                db.prepare(`CREATE VIRTUAL TABLE memory_vec USING vec0(
+						embedding float[${options.embeddingDimensions}] distance_metric=cosine
+					)`).run();
+            }
+            if (previousCapability === "blob-js") {
+                db.prepare("DELETE FROM memory_vec").run();
+                db.prepare("INSERT INTO memory_vec(rowid, embedding) SELECT id, embedding FROM memory_chunks").run();
+            }
+            // Virtual tables cannot own FK constraints, so this mirrors ON DELETE
+            // CASCADE for direct memory_chunks deletes while sqlite-vec is loaded.
+            db.prepare(`CREATE TRIGGER IF NOT EXISTS trg_memory_chunks_delete_vec
+				 AFTER DELETE ON memory_chunks
+				 BEGIN
+					DELETE FROM memory_vec WHERE rowid = old.id;
+				 END`).run();
+        }).immediate();
+        return "sqlite-vec";
+    }
+    catch {
+        db.prepare("DROP TRIGGER IF EXISTS trg_memory_chunks_delete_vec").run();
+        return "blob-js";
+    }
+}
+function migrateMemoryIndexSources(db) {
+    const columns = db.prepare("PRAGMA table_info(memory_chunks)").all();
+    const hasSourceColumns = columns.some((column) => column.name === "source_id");
+    if (hasSourceColumns) {
+        db.transaction(() => {
+            db.prepare(`INSERT OR IGNORE INTO memory_index_sources(chunk_id, corpus, source_id, source_ref, chunk_index)
+				 SELECT id, corpus, source_id, source_ref, chunk_index
+				 FROM memory_chunks
+				 WHERE source_id IS NOT NULL`).run();
+        }).immediate();
+    }
+    const ftsSql = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_fts'").get();
+    if (ftsSql && (ftsSql.sql.includes("content='memory_chunks'") || !ftsSql.sql.includes("contentless_delete=1"))) {
+        db.transaction(() => {
+            db.prepare("DROP TABLE memory_fts").run();
+            db.prepare(`CREATE VIRTUAL TABLE memory_fts USING fts5(
+					text_full,
+					snippet,
+					content='',
+					contentless_delete=1
+				)`).run();
+            const rows = db.prepare("SELECT id, text_full, snippet FROM memory_chunks").all();
+            const insert = db.prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)");
+            for (const row of rows)
+                insert.run(row.id, row.text_full, row.snippet);
+        }).immediate();
+    }
+}
+function tableExists(db, name) {
+    const row = db.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+    return !!row;
+}