@sigrun/recall 0.0.0-stage → 0.1.1

package/dist/tools.js

@@ -0,0 +1,174 @@
+import { Type } from "@earendil-works/pi-ai";
+import { defineTool } from "@earendil-works/pi-durable";
+import { audienceCovers } from "./audience.js";
+import { mayRead } from "./policy.js";
+import { redact } from "./redact.js";
+const OPEN_LIMIT = 16_384;
+function output(text) {
+    return { output: [{ type: "text", text }] };
+}
+function date(value) {
+    if (value === undefined)
+        return undefined;
+    const parsed = Date.parse(value);
+    return Number.isNaN(parsed) ? undefined : parsed;
+}
+function taint(db, conversationId, item) {
+    if (item.cls === "local-only") {
+        db.setTaint(conversationId, (db.cursor(conversationId) ?? 0) + 1);
+    }
+}
+async function permitted(deps, current, item, provider) {
+    if (!audienceCovers(current, { owner: item.owner ?? undefined }))
+        return undefined;
+    const meta = await deps.meta(item.conversationId).catch(() => undefined);
+    if (meta === undefined || meta.internal === true || !audienceCovers(current, meta)) {
+        return undefined;
+    }
+    const providers = item.owner === null ? [] : await deps.localProviders(item.owner).catch(() => []);
+    if (!mayRead(item.cls, provider === undefined ? undefined : { provider }, providers)) {
+        return undefined;
+    }
+    return meta;
+}
+function renderEntries(entries) {
+    const lines = [];
+    for (const entry of entries) {
+        for (const message of entry.model ?? []) {
+            if (message.role !== "user" && message.role !== "assistant" && message.role !== "toolResult") {
+                continue;
+            }
+            const text = typeof message.content === "string"
+                ? message.content
+                : message.content
+                    .filter((part) => part.type === "text")
+                    .map((part) => part.text)
+                    .join("\n");
+            if (text !== "")
+                lines.push(`${message.role}: ${redact(text)}`);
+        }
+    }
+    let bytes = 0;
+    const bounded = [];
+    for (const line of lines) {
+        if (bytes >= OPEN_LIMIT)
+            break;
+        const remaining = OPEN_LIMIT - bytes;
+        const buffer = Buffer.from(line, "utf8");
+        bounded.push(buffer.subarray(0, remaining).toString("utf8"));
+        bytes += Math.min(buffer.length, remaining);
+    }
+    return bounded.join("\n");
+}
+/** Build the read-only model-facing recall tools. */
+export function createTools(deps) {
+    const search = defineTool({
+        name: "recall_search",
+        description: "Search past conversations and stored tool results for relevant material.",
+        parameters: Type.Object({
+            query: Type.String(),
+            since: Type.Optional(Type.String()),
+            until: Type.Optional(Type.String()),
+            kind: Type.Optional(Type.Union([Type.Literal("conversation"), Type.Literal("page")])),
+            conversation: Type.Optional(Type.Number()),
+        }),
+        replay: "safe",
+        async execute(args, api, context) {
+            const conversationId = Number(api.conversationId);
+            const current = await deps.meta(conversationId);
+            const model = (await api.agent(context)).model;
+            const kinds = args.kind === "conversation"
+                ? ["turn", "tool"]
+                : args.kind === "page"
+                    ? ["artifact"]
+                    : undefined;
+            const hits = await deps.searcher.search(args.query, {
+                current: {
+                    conversationId,
+                    owner: current.owner,
+                    sharing: current.sharing,
+                },
+                model: model === undefined ? undefined : { provider: model.provider },
+                includeJobs: true,
+                since: date(args.since),
+                until: date(args.until),
+                kinds,
+            }, args.conversation === undefined ? 10 : 200);
+            const selected = hits
+                .filter((hit) => args.conversation === undefined || hit.item.conversationId === args.conversation)
+                .slice(0, 10);
+            for (const hit of selected)
+                taint(deps.db, conversationId, hit.item);
+            if (selected.length === 0)
+                return output("No matching past material.");
+            // Stable score order within first-seen conversation groups.
+            const groups = new Map();
+            for (const hit of selected) {
+                const group = groups.get(hit.item.conversationId) ?? [];
+                group.push(hit);
+                groups.set(hit.item.conversationId, group);
+            }
+            const lines = [...groups.values()].flatMap((group) => group.map((hit) => {
+                const day = new Date(hit.item.at).toISOString().slice(0, 10);
+                const title = hit.conversationTitle ?? `conversation ${hit.item.conversationId}`;
+                return `[${day}] ${title} — ${hit.snippet} (${hit.handle})`;
+            }));
+            return output(lines.join("\n"));
+        },
+    });
+    const open = defineTool({
+        name: "recall_open",
+        description: "Open a recalled passage or read a slice of a stored artifact.",
+        parameters: Type.Object({
+            handle: Type.String(),
+            offset: Type.Optional(Type.Number()),
+            length: Type.Optional(Type.Number({ maximum: OPEN_LIMIT })),
+        }),
+        replay: "safe",
+        async execute(args, api, context) {
+            const conversationId = Number(api.conversationId);
+            const current = await deps.meta(conversationId);
+            const model = (await api.agent(context)).model;
+            const provider = model?.provider;
+            const item = await itemForHandle(deps.db, args.handle);
+            if (item === undefined || (await permitted(deps, current, item, provider)) === undefined) {
+                return output("Not available here.");
+            }
+            taint(deps.db, conversationId, item);
+            if (args.handle.startsWith("a:")) {
+                const length = Math.min(args.length ?? OPEN_LIMIT, OPEN_LIMIT);
+                const text = deps.artifactStore.read(args.handle, args.offset ?? 0, length);
+                return text === undefined ? output("Not available here.") : output(text);
+            }
+            if (item.entryId === null)
+                return output("Not available here.");
+            const page = await deps
+                .entries(item.conversationId, {
+                minEntryId: Math.max(1, item.entryId - 3),
+                maxEntryId: item.entryId + 3,
+                order: "ascending",
+            }, 10, undefined)
+                .catch(() => undefined);
+            if (page === undefined)
+                return output("Not available here.");
+            const nearby = deps.db.itemsForEntries(item.conversationId, page.items.map((entry) => entry.id));
+            for (const candidate of nearby) {
+                if ((await permitted(deps, current, candidate, provider)) === undefined) {
+                    return output("Not available here.");
+                }
+            }
+            for (const candidate of nearby)
+                taint(deps.db, conversationId, candidate);
+            const text = renderEntries(page.items);
+            return output(text === "" ? "Not available here." : text);
+        },
+    });
+    return [search, open];
+}
+async function itemForHandle(db, handle) {
+    if (handle.startsWith("r:"))
+        return db.item(Number(handle.slice(2)));
+    if (handle.startsWith("a:"))
+        return db.itemByArtifact(handle);
+    return undefined;
+}

package/dist/vectors.d.ts

@@ -0,0 +1,17 @@
+import type { Embedder } from "./embed.js";
+import type { RecallDb } from "./db.js";
+import type { EntryRecord } from "@earendil-works/pi-durable";
+import type { VectorArm } from "./search.js";
+export interface VectorIndex {
+    upsert(itemId: number, vec: Float32Array): void;
+    query(vec: Float32Array, k: number, signal?: AbortSignal): {
+        itemId: number;
+        score: number;
+    }[];
+}
+/** Store vectors in SQLite and scan bounded pages for cosine nearest neighbours. */
+export declare function createSqliteVectorIndex(db: RecallDb, pageSize?: number): VectorIndex;
+/** Embed up to `limit` unembedded conversation passages from their source entries. */
+export declare function embedPending(db: RecallDb, embedder: Embedder, index: VectorIndex, entry: (conversationId: number, entryId: number) => Promise<EntryRecord | undefined>, limit?: number): Promise<void>;
+/** Add a semantic arm that degrades to lexical search on embedding failure. */
+export declare function createVectorArm(embedder: Embedder, index: VectorIndex, timeoutMs: number): VectorArm;

package/dist/vectors.js

@@ -0,0 +1,105 @@
+import { passagesOf } from "./passages.js";
+import { redact } from "./redact.js";
+/** Store vectors in SQLite and scan bounded pages for cosine nearest neighbours. */
+export function createSqliteVectorIndex(db, pageSize = 2000) {
+    return {
+        upsert(itemId, vec) {
+            db.upsertVector(itemId, vec);
+        },
+        query(vec, k, signal) {
+            if (k <= 0)
+                return [];
+            const best = [];
+            let after = 0;
+            for (let page = 0; page < 50; page += 1) {
+                if (signal?.aborted === true)
+                    break;
+                const rows = db.listVectorsPage(after, pageSize);
+                for (const row of rows) {
+                    const score = cosine(vec, row.vec);
+                    best.push({ itemId: row.itemId, score });
+                    best.sort((a, b) => b.score - a.score);
+                    if (best.length > k)
+                        best.pop();
+                }
+                if (rows.length < pageSize)
+                    break;
+                after = rows.at(-1).itemId;
+            }
+            return best;
+        },
+    };
+}
+function cosine(left, right) {
+    const rightVector = new Float32Array(right.buffer.slice(right.byteOffset, right.byteOffset + right.byteLength));
+    if (left.length !== rightVector.length)
+        return -1;
+    let dot = 0;
+    let leftNorm = 0;
+    let rightNorm = 0;
+    for (let index = 0; index < left.length; index += 1) {
+        dot += left[index] * rightVector[index];
+        leftNorm += left[index] ** 2;
+        rightNorm += rightVector[index] ** 2;
+    }
+    if (leftNorm === 0 || rightNorm === 0)
+        return 0;
+    return dot / Math.sqrt(leftNorm * rightNorm);
+}
+/** Embed up to `limit` unembedded conversation passages from their source entries. */
+export async function embedPending(db, embedder, index, entry, limit = 500) {
+    const candidates = [];
+    for (const item of db.listUnembeddedItems(limit, embedder.local)) {
+        if (item.entryId === null ||
+            item.messageIndex === null) {
+            continue;
+        }
+        const record = await entry(item.conversationId, item.entryId);
+        if (record === undefined)
+            continue;
+        const passage = passagesOf(record).find((candidate) => candidate.messageIndex === item.messageIndex && candidate.offset === item.offset);
+        if (passage !== undefined)
+            candidates.push({ id: item.id, text: redact(passage.text) });
+    }
+    for (let offset = 0; offset < candidates.length; offset += 64) {
+        const batch = candidates.slice(offset, offset + 64);
+        try {
+            const vectors = await embedder.embed(batch.map(({ text }) => text));
+            for (const [position, vector] of vectors.entries()) {
+                const candidate = batch[position];
+                if (candidate !== undefined)
+                    index.upsert(candidate.id, vector);
+            }
+        }
+        catch (error) {
+            console.error("recall: pending passage embedding failed", error);
+        }
+    }
+}
+/** Add a semantic arm that degrades to lexical search on embedding failure. */
+export function createVectorArm(embedder, index, timeoutMs) {
+    let lastWarning = -Infinity;
+    return {
+        async query(text, candidateLimit) {
+            const signal = AbortSignal.timeout(timeoutMs);
+            try {
+                const vectors = await embedder.embed([text], signal);
+                const vector = vectors[0];
+                if (vector === undefined)
+                    return [];
+                return index.query(vector, candidateLimit, signal).map(({ itemId, score }) => ({
+                    itemId,
+                    rank: score,
+                }));
+            }
+            catch (error) {
+                const now = Date.now();
+                if (now - lastWarning >= 60_000) {
+                    console.warn("recall: semantic search unavailable; using lexical results", error);
+                    lastWarning = now;
+                }
+                return [];
+            }
+        },
+    };
+}

package/package.json

@@ -1,6 +1,57 @@
 {
-  "name": "@sigrun/recall",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+	"name": "@sigrun/recall",
+	"version": "0.1.1",
+	"description": "Cross-conversation recall extension for sigrun (pi-durable)",
+	"type": "module",
+	"engines": {
+		"node": ">=22.19.0"
+	},
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"default": "./dist/index.js"
+		}
+	},
+	"files": [
+		"dist",
+		"CHANGELOG.md"
+	],
+	"scripts": {
+		"test": "node scripts/check-manifest.mjs --expect-name @sigrun/recall && vitest run",
+		"typecheck": "tsc --noEmit",
+		"build": "tsc -p tsconfig.build.json",
+		"prepare": "npm run build"
+	},
+	"peerDependencies": {
+		"@earendil-works/chord": "1.4.0",
+		"@earendil-works/pi-ai": "1.4.0",
+		"@earendil-works/pi-durable": "1.4.0",
+		"@sigrun/shared": "0.5.0"
+	},
+	"devDependencies": {
+		"@earendil-works/chord": "1.4.0",
+		"@earendil-works/pi-ai": "1.4.0",
+		"@earendil-works/pi-durable": "1.4.0",
+		"@sigrun/sigrun": "file:../sigrun",
+		"@types/node": "^26.6.4",
+		"tsx": "^4.23.15",
+		"typescript": "^7.0.2",
+		"vitest": "^5.0.3",
+		"@sigrun/shared": "file:../sigrun/shared"
+	},
+	"license": "MIT",
+	"repository": {
+		"type": "github",
+		"url": "git+https://github.com/sigrun-ai/sigrun-recall.git"
+	},
+	"bugs": {
+		"url": "https://github.com/sigrun-ai/sigrun-recall/issues"
+	},
+	"homepage": "https://github.com/sigrun-ai/sigrun-recall#readme",
+	"publishConfig": {
+		"access": "public",
+		"provenance": true
+	}
+}