@sigrun/recall 0.0.0-stage → 0.1.1

package/dist/db.js

@@ -0,0 +1,306 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+const SCHEMA = `
+	CREATE TABLE IF NOT EXISTS items (
+		id INTEGER PRIMARY KEY,
+		conversationId INTEGER NOT NULL,
+		owner TEXT,
+		kind TEXT NOT NULL CHECK (kind IN ('turn', 'tool', 'artifact')),
+		entryId INTEGER,
+		messageIndex INTEGER,
+		blockIndex INTEGER NOT NULL DEFAULT 0,
+		offset INTEGER NOT NULL,
+		length INTEGER NOT NULL,
+		artifact TEXT,
+		job INTEGER NOT NULL CHECK (job IN (0, 1)),
+		cls TEXT NOT NULL CHECK (cls IN ('normal', 'local-only')),
+		at INTEGER NOT NULL,
+		title TEXT
+	);
+	CREATE UNIQUE INDEX IF NOT EXISTS items_message_key
+		ON items (conversationId, kind, entryId, messageIndex, blockIndex, offset)
+		WHERE kind IN ('turn', 'tool');
+	CREATE UNIQUE INDEX IF NOT EXISTS items_artifact_key
+		ON items (artifact, offset)
+		WHERE kind = 'artifact';
+
+	CREATE VIRTUAL TABLE IF NOT EXISTS fts_porter
+		USING fts5(text, tokenize='porter unicode61', content='');
+	CREATE VIRTUAL TABLE IF NOT EXISTS fts_trigram
+		USING fts5(text, tokenize='trigram', content='');
+
+	CREATE TABLE IF NOT EXISTS artifacts (
+		hash TEXT PRIMARY KEY,
+		bytes INTEGER NOT NULL,
+		title TEXT,
+		url TEXT,
+		createdAt INTEGER NOT NULL
+	);
+	CREATE TABLE IF NOT EXISTS cursors (
+		conversationId INTEGER PRIMARY KEY,
+		entryId INTEGER NOT NULL
+	);
+	CREATE TABLE IF NOT EXISTS taint (
+		conversationId INTEGER PRIMARY KEY,
+		entryId INTEGER NOT NULL
+	);
+	CREATE TABLE IF NOT EXISTS injections (
+		conversationId INTEGER NOT NULL,
+		messageKey TEXT NOT NULL,
+		sources TEXT NOT NULL DEFAULT '[]',
+		block TEXT NOT NULL,
+		cls TEXT NOT NULL CHECK (cls IN ('normal', 'local-only')),
+		PRIMARY KEY (conversationId, messageKey)
+	);
+	CREATE TABLE IF NOT EXISTS vectors (
+		itemId INTEGER PRIMARY KEY,
+		dim INTEGER NOT NULL,
+		vec BLOB NOT NULL
+	);
+	CREATE TABLE IF NOT EXISTS state (
+		key TEXT PRIMARY KEY,
+		value TEXT NOT NULL
+	);
+	CREATE TABLE IF NOT EXISTS schema_version (
+		version INTEGER NOT NULL
+	);
+`;
+/** The extension's private SQLite index. Callers close it during shutdown. */
+export class RecallDb {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    static open(path) {
+        if (path !== ":memory:")
+            mkdirSync(dirname(path), { recursive: true });
+        const db = new DatabaseSync(path);
+        try {
+            const migrated = new RecallDb(db);
+            migrated.migrate();
+            return migrated;
+        }
+        catch (error) {
+            db.close();
+            throw error;
+        }
+    }
+    migrate() {
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            this.db.exec("CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)");
+            const current = this.db
+                .prepare("SELECT version FROM schema_version")
+                .get();
+            if (current === undefined) {
+                this.db.exec(SCHEMA);
+                this.db.prepare("INSERT INTO schema_version (version) VALUES (2)").run();
+            }
+            else if (current.version < 2) {
+                const itemColumns = this.db.prepare("PRAGMA table_info(items)").all();
+                if (!itemColumns.some(({ name }) => name === "blockIndex")) {
+                    this.db.exec("ALTER TABLE items ADD COLUMN blockIndex INTEGER NOT NULL DEFAULT 0");
+                }
+                const injectionColumns = this.db.prepare("PRAGMA table_info(injections)").all();
+                if (!injectionColumns.some(({ name }) => name === "sources")) {
+                    this.db.exec("ALTER TABLE injections ADD COLUMN sources TEXT NOT NULL DEFAULT '[]'");
+                }
+                this.db.exec("DROP INDEX items_message_key");
+                this.db.exec(SCHEMA);
+                this.db.prepare("UPDATE schema_version SET version = 2").run();
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    insertItem(row, text) {
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            const inserted = this.db
+                .prepare(`INSERT INTO items (
+					conversationId, owner, kind, entryId, messageIndex, blockIndex, offset,
+					length, artifact, job, cls, at, title
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+
+					ON CONFLICT(conversationId, kind, entryId, messageIndex, blockIndex, offset)
+						WHERE kind IN ('turn', 'tool') DO NOTHING
+					ON CONFLICT(artifact, offset)
+						WHERE kind = 'artifact' DO NOTHING
+					RETURNING id`)
+                .get(row.conversationId, row.owner, row.kind, row.entryId, row.messageIndex, row.blockIndex, row.offset, row.length, row.artifact, row.job ? 1 : 0, row.cls, row.at, row.title);
+            if (inserted === undefined) {
+                this.db.exec("COMMIT");
+                return undefined;
+            }
+            this.db
+                .prepare("INSERT INTO fts_porter(rowid, text) VALUES (?, ?)")
+                .run(inserted.id, text);
+            this.db
+                .prepare("INSERT INTO fts_trigram(rowid, text) VALUES (?, ?)")
+                .run(inserted.id, text);
+            this.db.exec("COMMIT");
+            return inserted.id;
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    putArtifact(hash, bytes, meta) {
+        this.db
+            .prepare(`INSERT INTO artifacts (hash, bytes, title, url, createdAt)
+				 VALUES (?, ?, ?, ?, ?) ON CONFLICT(hash) DO NOTHING`)
+            .run(hash, bytes, meta.title ?? null, meta.url ?? null, meta.createdAt);
+    }
+    item(id) {
+        const row = this.db.prepare("SELECT * FROM items WHERE id = ?").get(id);
+        return row === undefined ? undefined : { ...row, job: row.job === 1 };
+    }
+    itemByPassage(conversationId, entryId, messageIndex, blockIndex, offset) {
+        const row = this.db
+            .prepare(`SELECT * FROM items WHERE conversationId = ? AND entryId = ?
+				 AND messageIndex = ? AND blockIndex = ? AND offset = ? AND kind IN ('turn', 'tool')`)
+            .get(conversationId, entryId, messageIndex, blockIndex, offset);
+        return row === undefined ? undefined : { ...row, job: row.job === 1 };
+    }
+    itemsForEntries(conversationId, entryIds) {
+        if (entryIds.length === 0)
+            return [];
+        const placeholders = entryIds.map(() => "?").join(",");
+        // SAFETY: placeholders are generated from array length; values are bound.
+        const rows = this.db
+            .prepare(`SELECT * FROM items WHERE conversationId = ? AND entryId IN (${placeholders})`)
+            .all(conversationId, ...entryIds);
+        return rows.map((row) => ({ ...row, job: row.job === 1 }));
+    }
+    artifactHash(handle) {
+        const row = this.db
+            .prepare("SELECT hash FROM artifacts WHERE substr(hash, 1, 16) = ?")
+            .get(handle.slice(2));
+        return row?.hash;
+    }
+    itemByArtifact(handle) {
+        const row = this.db
+            .prepare("SELECT * FROM items WHERE artifact = ? ORDER BY id LIMIT 1")
+            .get(handle);
+        return row === undefined ? undefined : { ...row, job: row.job === 1 };
+    }
+    searchPorter(query, limit) {
+        return this.searchFts("fts_porter", query, limit);
+    }
+    searchTrigram(query, limit) {
+        return this.searchFts("fts_trigram", query, limit);
+    }
+    searchFts(table, query, limit) {
+        // SAFETY: table is an internal two-value union; values are bound parameters.
+        // The query selects one integer id and one numeric bm25 rank per row.
+        const rows = this.db
+            .prepare(`SELECT rowid AS id, bm25(${table}) AS rank FROM ${table}
+				 WHERE ${table} MATCH ? ORDER BY rank LIMIT ?`)
+            .all(query, limit);
+        return rows.map(({ id, rank }) => ({ id, bm25: rank }));
+    }
+    /** Indexed text for one item from the porter FTS table. */
+    itemText(id) {
+        const row = this.db
+            .prepare("SELECT text FROM fts_porter WHERE rowid = ?")
+            .get(id);
+        return row?.text;
+    }
+    /** Item ids matching a trigram FTS5 expression, in match order. */
+    matchTrigram(expression) {
+        // SAFETY: node:sqlite returns plain row objects; this query selects one integer.
+        const rows = this.db
+            .prepare("SELECT rowid FROM fts_trigram WHERE fts_trigram MATCH ?")
+            .all(expression);
+        return rows.map((row) => row.rowid);
+    }
+    upsertVector(itemId, vec) {
+        const bytes = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
+        this.db
+            .prepare(`INSERT INTO vectors (itemId, dim, vec) VALUES (?, ?, ?)
+				 ON CONFLICT(itemId) DO UPDATE SET dim = excluded.dim, vec = excluded.vec`)
+            .run(itemId, vec.length, bytes);
+    }
+    listVectorsPage(after, limit) {
+        // SAFETY: the vectors schema stores integer item ids and Float32 BLOB bytes.
+        return this.db
+            .prepare("SELECT itemId, vec FROM vectors WHERE itemId > ? ORDER BY itemId LIMIT ?")
+            .all(after, limit);
+    }
+    state(key) {
+        const row = this.db.prepare("SELECT value FROM state WHERE key = ?").get(key);
+        return row?.value;
+    }
+    setState(key, value) {
+        this.db
+            .prepare("INSERT INTO state (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
+            .run(key, value);
+    }
+    listUnembeddedItems(limit, includeLocalOnly) {
+        // SAFETY: the LEFT JOIN projects the typed items schema only.
+        const rows = this.db
+            .prepare(`SELECT items.* FROM items LEFT JOIN vectors ON vectors.itemId = items.id
+				 WHERE vectors.itemId IS NULL AND items.kind IN ('turn', 'tool')
+				 AND items.entryId IS NOT NULL AND items.messageIndex IS NOT NULL
+				 AND (? OR items.cls != 'local-only') ORDER BY items.id LIMIT ?`)
+            .all(includeLocalOnly ? 1 : 0, limit);
+        return rows.map((row) => ({ ...row, job: row.job === 1 }));
+    }
+    cursor(conversationId) {
+        return this.entryId("cursors", conversationId);
+    }
+    setCursor(conversationId, entryId) {
+        this.setEntryId("cursors", conversationId, entryId);
+    }
+    taintFrom(conversationId) {
+        return this.entryId("taint", conversationId);
+    }
+    setTaint(conversationId, entryId) {
+        // The earliest point wins: everything from it onward stays local-only.
+        this.db
+            .prepare(`INSERT INTO taint (conversationId, entryId) VALUES (?, ?)
+				ON CONFLICT(conversationId) DO UPDATE SET entryId = excluded.entryId
+				WHERE excluded.entryId < taint.entryId`)
+            .run(conversationId, entryId);
+    }
+    entryId(table, conversationId) {
+        const row = this.db
+            .prepare(`SELECT entryId FROM ${table} WHERE conversationId = ?`)
+            .get(conversationId);
+        return row?.entryId;
+    }
+    setEntryId(table, conversationId, entryId) {
+        this.db
+            .prepare(`INSERT INTO ${table} (conversationId, entryId) VALUES (?, ?)
+				ON CONFLICT(conversationId) DO UPDATE SET entryId = excluded.entryId`)
+            .run(conversationId, entryId);
+    }
+    putInjection(conversationId, messageKey, block, cls, sources = []) {
+        this.db
+            .prepare(`INSERT INTO injections (conversationId, messageKey, block, cls, sources) VALUES (?, ?, ?, ?, ?)
+				ON CONFLICT(conversationId, messageKey) DO UPDATE SET
+					block = excluded.block, cls = excluded.cls, sources = excluded.sources`)
+            .run(conversationId, messageKey, block, cls, JSON.stringify(sources));
+    }
+    injections(conversationId) {
+        // SAFETY: these rows come from the typed injections schema above.
+        const rows = this.db
+            .prepare("SELECT messageKey, block, cls, sources FROM injections WHERE conversationId = ?")
+            .all(conversationId);
+        return new Map(rows.map((row) => {
+            const parsed = JSON.parse(row.sources);
+            const sources = Array.isArray(parsed)
+                ? parsed.filter((id) => Number.isInteger(id) && id > 0)
+                : [];
+            return [row.messageKey, { block: row.block, cls: row.cls, sources }];
+        }));
+    }
+    close() {
+        this.db.close();
+    }
+}

package/dist/embed.d.ts

@@ -0,0 +1,10 @@
+export interface Embedder {
+    embed(texts: readonly string[], signal?: AbortSignal): Promise<Float32Array[]>;
+    readonly local: boolean;
+}
+/** Call an [OI]-compatible embeddings endpoint without exposing a key by default. */
+export declare function createEmbedder(cfg: {
+    url: string;
+    model: string;
+    local: boolean;
+}, apiKey: () => Promise<string | undefined>, fetchImpl?: typeof fetch): Embedder;

package/dist/embed.js

@@ -0,0 +1,22 @@
+/** Call an [OI]-compatible embeddings endpoint without exposing a key by default. */
+export function createEmbedder(cfg, apiKey, fetchImpl = fetch) {
+    return {
+        local: cfg.local,
+        async embed(texts, signal) {
+            const key = await apiKey();
+            const headers = { "content-type": "application/json" };
+            if (typeof key === "string" && key !== "")
+                headers.authorization = `Bearer ${key}`;
+            const response = await fetchImpl(`${cfg.url}/embeddings`, {
+                method: "POST",
+                headers,
+                body: JSON.stringify({ model: cfg.model, input: texts }),
+                signal,
+            });
+            if (!response.ok)
+                throw new Error(`embedding endpoint returned ${response.status}`);
+            const body = (await response.json());
+            return body.data.map(({ embedding }) => new Float32Array(embedding));
+        },
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+import type { SigrunExtensionResult, SigrunHost } from "@sigrun/sigrun/extension";
+import { RecallDb } from "./db.js";
+export interface FactoryDeps {
+    openDb?: (path: string) => RecallDb;
+    now?: () => number;
+}
+/**
+ * The loader injects the host only when factory.length > 0: keep the host
+ * bare and the optional test dependencies second, so the arity stays one.
+ */
+export default function createRecall(host: SigrunHost, deps?: FactoryDeps): SigrunExtensionResult;

package/dist/index.js

@@ -0,0 +1,215 @@
+import { join } from "node:path";
+import { AgentDoc, defineExtension, GenerationTask, hook, ToolTask, } from "@earendil-works/pi-durable";
+import { createArtifactStore, externalize } from "./artifacts.js";
+import { BACKFILL_TASK_NAME, createBackfillTask, runSweep } from "./backfill.js";
+import { captureConversation } from "./capture.js";
+import { resolveSettings, resolveUserSettings } from "./config.js";
+import { RecallDb } from "./db.js";
+import { createSearcher } from "./search.js";
+import { createRoutes } from "./route.js";
+import { createTools } from "./tools.js";
+import { createEmbedder } from "./embed.js";
+import { applyRecall } from "./inject.js";
+import { createSqliteVectorIndex, createVectorArm, embedPending } from "./vectors.js";
+/**
+ * The loader injects the host only when factory.length > 0: keep the host
+ * bare and the optional test dependencies second, so the arity stays one.
+ */
+export default function createRecall(host, deps = {}) {
+    let database;
+    let embedder;
+    let embeddingKey;
+    let services;
+    const pending = new Set();
+    const trackEmbedding = (work) => {
+        pending.add(work);
+        void work.finally(() => pending.delete(work));
+    };
+    // Capture/search surfaces added by later tasks share this one lazy handle.
+    // Core does not create paths.data; RecallDb.open creates its parent.
+    function db() {
+        database ??= (deps.openDb ?? RecallDb.open)(join(host.stores.paths.data, "recall.db"));
+        return database;
+    }
+    async function semantic() {
+        const settings = await resolveSettings(host.stores.settings);
+        if (settings.embedding === undefined)
+            return {};
+        const key = JSON.stringify(settings.embedding);
+        if (embedder === undefined || embeddingKey !== key) {
+            embedder = createEmbedder(settings.embedding, () => host.stores.secrets.get("embedding-api-key"));
+            embeddingKey = key;
+        }
+        return {
+            embedder,
+            index: createSqliteVectorIndex(db()),
+            timeoutMs: settings.embedding.timeoutMs,
+        };
+    }
+    async function vectors() {
+        const semanticDeps = await semantic();
+        if (semanticDeps.embedder === undefined ||
+            semanticDeps.index === undefined ||
+            semanticDeps.timeoutMs === undefined) {
+            return undefined;
+        }
+        return createVectorArm(semanticDeps.embedder, semanticDeps.index, semanticDeps.timeoutMs);
+    }
+    const artifactStore = () => createArtifactStore(db(), join(host.stores.paths.data, "artifacts"));
+    const meta = (id) => host.conversations.meta(id);
+    const entry = (conversationId, entryId) => host.conversations.entry(conversationId, entryId);
+    const userSettings = async (user) => {
+        try {
+            return await resolveUserSettings(host.stores.user(user).settings);
+        }
+        catch {
+            return { autoRecall: true, localProviders: [] };
+        }
+    };
+    const localProviders = async (user) => (await userSettings(user)).localProviders;
+    const pendingVectors = async () => {
+        const semanticDeps = await semantic();
+        if (semanticDeps.embedder !== undefined && semanticDeps.index !== undefined) {
+            await embedPending(db(), semanticDeps.embedder, semanticDeps.index, entry, 500);
+        }
+    };
+    const backfillDeps = () => ({
+        host,
+        capture: captureConversation,
+        db: db(),
+        vectors: pendingVectors,
+        now: deps.now ?? Date.now,
+    });
+    function getServices() {
+        if (services !== undefined)
+            return services;
+        const vectorIndex = createSqliteVectorIndex(db());
+        const searcher = createSearcher({
+            db: db(),
+            meta,
+            entry,
+            userSettings,
+            artifactStore: artifactStore(),
+            vectors: {
+                query: async (text, limit) => (await vectors())?.query(text, limit) ?? [],
+            },
+            now: deps.now ?? Date.now,
+        });
+        services = {
+            searcher,
+            tools: createTools({
+                db: db(),
+                searcher,
+                artifactStore: artifactStore(),
+                meta,
+                entries: (conversationId, query, limit, cursor) => host.conversations.entries(conversationId, {
+                    ...query,
+                    minEntryId: query.minEntryId,
+                    maxEntryId: query.maxEntryId,
+                }, limit, cursor),
+                localProviders,
+            }),
+        };
+        void vectorIndex;
+        return services;
+    }
+    const extension = {
+        name: "@sigrun/recall",
+        hooks: [
+            hook(ToolTask, {
+                afterTool: async (call, result, api) => {
+                    try {
+                        const meta = await host.conversations.meta(api.conversationId);
+                        if (meta.internal === true)
+                            return undefined;
+                        const settings = await resolveSettings(host.stores.settings);
+                        return externalize(result, call, {
+                            store: artifactStore(),
+                            thresholdBytes: settings.externalizeBytes,
+                            conversationId: api.conversationId,
+                            owner: meta.owner ?? null,
+                            now: (deps.now ?? Date.now)(),
+                        });
+                    }
+                    catch (error) {
+                        console.error("recall: tool-result externalization failed", error);
+                        return undefined;
+                    }
+                },
+            }),
+            hook(GenerationTask, {
+                beforeRequest: async (request, api, context) => {
+                    try {
+                        const liveMeta = await host.conversations.meta(api.conversationId);
+                        const settings = liveMeta.owner === undefined
+                            ? { autoRecall: true, localProviders: [] }
+                            : await userSettings(liveMeta.owner);
+                        const agent = await api.snapshot(AgentDoc, api.conversationId, context);
+                        const messages = await applyRecall(request.messages, {
+                            conversationId: api.conversationId,
+                            db: db(),
+                            searcher: getServices().searcher,
+                            meta: liveMeta,
+                            model: agent?.model,
+                            autoRecall: settings.autoRecall,
+                            internal: liveMeta.internal === true,
+                            ownerLocalProviders: settings.localProviders,
+                            sourceMeta: (id) => meta(id),
+                            sourceLocalProviders: localProviders,
+                        });
+                        return messages === undefined ? undefined : { messages };
+                    }
+                    catch (error) {
+                        console.error("recall: automatic recall failed", error);
+                        return undefined;
+                    }
+                },
+            }),
+        ],
+        get tools() {
+            return getServices().tools;
+        },
+        get tasks() {
+            return [createBackfillTask(backfillDeps)];
+        },
+    };
+    return {
+        extension: defineExtension(extension),
+        jobs: [
+            {
+                name: "backfill",
+                schedule: { cron: "*/10 * * * *" },
+                action: { kind: "task", task: BACKFILL_TASK_NAME },
+            },
+        ],
+        routes: createRoutes({
+            searcher: {
+                search: (query, context, limit) => getServices().searcher.search(query, context, limit),
+            },
+            sweep: () => runSweep(backfillDeps()),
+        }),
+        runSettled: async (event, api, context) => {
+            try {
+                const semanticDeps = await semantic();
+                await captureConversation({
+                    db: db(),
+                    meta: (id) => host.conversations.meta(id),
+                    now: deps.now ?? Date.now,
+                    ...semanticDeps,
+                    trackEmbedding,
+                }, event.conversationId, (query, limit, cursor) => api.entries({ ...query, minEntryId: query.minEntryId }, limit, cursor, context));
+            }
+            catch (error) {
+                console.error("recall: conversation capture failed", error);
+            }
+        },
+        harnessShutdown: async () => {
+            const deadline = Date.now() + 30_000;
+            while (pending.size > 0 && Date.now() < deadline) {
+                await Promise.allSettled([...pending]);
+            }
+            database?.close();
+            database = undefined;
+        },
+    };
+}

package/dist/inject.d.ts

@@ -0,0 +1,25 @@
+import type { Message, UserMessage } from "@earendil-works/pi-ai";
+import type { SessionMeta } from "@sigrun/shared/session-meta";
+import type { RecallDb } from "./db.js";
+import type { Hit, Searcher } from "./search.js";
+export interface InjectDeps {
+    conversationId: number;
+    db: RecallDb;
+    searcher: Searcher;
+    meta: SessionMeta;
+    model: {
+        provider: string;
+    } | undefined;
+    autoRecall: boolean;
+    internal: boolean;
+    ownerLocalProviders: readonly string[];
+    /** Live source policy readers; missing readers fail closed for sourced blocks. */
+    sourceMeta?: (conversationId: number) => Promise<SessionMeta>;
+    sourceLocalProviders?: (user: string) => Promise<readonly string[]>;
+}
+/** Stable identity of a user message for per-message injection reuse. */
+export declare function messageKey(message: UserMessage): string;
+/** Render stable, untrusted reference material within the snippet byte budget. */
+export declare function renderBlock(hits: readonly Hit[]): string;
+/** Search once per newest message and apply all currently readable stored blocks. */
+export declare function applyRecall(messages: readonly Message[], deps: InjectDeps): Promise<readonly Message[] | undefined>;