@topodb/pi 0.0.1 → 0.0.3

package/dist/extension.js

@@ -1,7 +1,14 @@
 import { Type } from "typebox";
-import { TopodbServer } from "./server-handle.js";
+import { TopodbServer, recordingEnabled } from "./server-handle.js";
+import { EpisodeBuffer, extractText, isUsed, buildEpisodeBatch, toRetrievalRecord } from "./recorder.js";
+import { ensurePolicyVersion } from "./policy.js";
 export default function (pi) {
     const server = new TopodbServer();
+    const recording = recordingEnabled(process.env);
+    const buffer = new EpisodeBuffer();
+    const memContents = new Map(); // memory id -> content seen at retrieval
+    let policyId;
+    let policyResolved = false;
     pi.registerTool({
         name: "topodb",
         label: "topodb memory",
@@ -39,6 +46,19 @@ export default function (pi) {
                     };
                 }
                 const result = await server.call(params.tool, params.args ?? {});
+                if (recording && buffer.open && (params.tool === "search_memories" || params.tool === "traverse")) {
+                    try {
+                        const cap = toRetrievalRecord(params.tool, params.args ?? {}, result);
+                        if (cap) {
+                            buffer.addRetrieval(cap.record);
+                            for (const [id, content] of cap.contents)
+                                memContents.set(id, content);
+                        }
+                    }
+                    catch (e) {
+                        console.error(`topodb recorder: retrieval capture failed: ${e.message}`);
+                    }
+                }
                 return { content: [{ type: "text", text: JSON.stringify(result) }], details: { tool: params.tool } };
             }
             catch (e) {
@@ -46,7 +66,108 @@ export default function (pi) {
             }
         },
     });
+    pi.on("agent_start", async () => {
+        if (!recording)
+            return;
+        try {
+            buffer.start(Date.now());
+            memContents.clear();
+            if (!policyResolved) {
+                policyResolved = true;
+                const paths = (process.env.TOPODB_POLICY_PATHS ?? "")
+                    .split(",")
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+                if (paths.length) {
+                    policyId = await ensurePolicyVersion((t, a) => server.call(t, a), paths);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`topodb recorder: agent_start failed: ${e.message}`);
+        }
+    });
+    pi.on("turn_end", async () => {
+        if (recording)
+            buffer.bumpTurns();
+    });
+    pi.on("tool_execution_end", async (ev) => {
+        if (recording && ev.isError)
+            buffer.noteToolError();
+    });
+    pi.on("agent_end", async (ev) => {
+        if (!recording || !buffer.open)
+            return;
+        try {
+            buffer.close();
+            const msgs = ev.messages ?? [];
+            const firstUser = msgs.find((m) => m.role === "user");
+            const goal = extractText(firstUser?.content).slice(0, 2000);
+            const assistants = msgs.filter((m) => m.role === "assistant");
+            const tokens = assistants.reduce((n, m) => n + (m.usage?.input ?? 0) + (m.usage?.output ?? 0), 0);
+            const last = assistants[assistants.length - 1];
+            const aborted = last?.stopReason === "aborted" || last?.stopReason === "error" || Boolean(last?.errorMessage);
+            const outcome = aborted || buffer.toolErrors > 0 ? "failure" : "success";
+            const failure = aborted
+                ? `run ended with stopReason=${last?.stopReason ?? "?"}`
+                : buffer.toolErrors > 0
+                    ? `${buffer.toolErrors} tool error(s)`
+                    : "";
+            const textAfter = (atMs) => assistants
+                .filter((m) => (m.timestamp ?? 0) >= atMs)
+                .map((m) => extractText(m.content))
+                .join("\n");
+            const used = new Map();
+            buffer.retrievals.forEach((r, i) => {
+                const hay = textAfter(r.at);
+                const s = new Set();
+                for (const m of r.returned) {
+                    const content = memContents.get(m.id) ?? "";
+                    if (isUsed(content, hay))
+                        s.add(m.id);
+                }
+                if (s.size)
+                    used.set(i, s);
+            });
+            const cmds = buildEpisodeBatch({
+                buffer,
+                goal,
+                outcome,
+                failure,
+                endedAt: Date.now(),
+                tokens,
+                used,
+                policyVersionId: policyId,
+            });
+            await server.call("submit_batch", { commands: cmds });
+        }
+        catch (e) {
+            console.error(`topodb recorder: episode write failed: ${e.message}`);
+        }
+    });
     pi.on("session_shutdown", async () => {
+        // An episode still open at shutdown (crash/quit mid-run) must be flushed
+        // as a failure before the server goes down, so the graph never has a
+        // dangling in-progress episode with no outcome recorded.
+        if (recording && buffer.open) {
+            try {
+                buffer.close();
+                const cmds = buildEpisodeBatch({
+                    buffer,
+                    goal: "",
+                    outcome: "failure",
+                    failure: "session shutdown mid-run",
+                    endedAt: Date.now(),
+                    tokens: 0,
+                    used: new Map(),
+                    policyVersionId: policyId,
+                });
+                await server.call("submit_batch", { commands: cmds });
+            }
+            catch {
+                /* dying anyway */
+            }
+        }
         server.shutdown();
     });
 }

package/dist/policy.js

@@ -0,0 +1,130 @@
+// src/policy.ts — optional PolicyVersion bootstrap: records which prompt/
+// config artifacts were in effect for a run. Hashes the configured artifact
+// files and find-or-creates the content-addressed Artifact nodes plus the
+// PolicyVersion node that INCLUDES exactly that set, so later analysis can
+// tell which policy version produced which episodes.
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+const INLINE_LIMIT = 64 * 1024;
+export function hashArtifacts(paths) {
+    return [...paths]
+        .sort()
+        .map((p) => {
+        const bytes = readFileSync(p);
+        return {
+            path: p.replace(/\\/g, "/"),
+            sha256: createHash("sha256").update(bytes).digest("hex"),
+            content: bytes.length <= INLINE_LIMIT ? bytes.toString("utf8") : "",
+            kind: p.endsWith(".md") ? "prompt" : "config",
+        };
+    });
+}
+/** Find-or-create the PolicyVersion for exactly this artifact set. Returns
+ * its node id, or undefined on ANY failure (recording must never break the
+ * agent). Wire protocol shapes follow topodb-mcp's JSON results; every
+ * access is defensive. */
+export async function ensurePolicyVersion(call, paths) {
+    try {
+        // Dedupe by sha256 (keep the first path per hash — hashArtifacts sorts
+        // by path). Everything downstream — the wanted set, the reuse gate, the
+        // create batch, the INCLUDES links — is driven off this deduped list;
+        // duplicate content must map to ONE Artifact node and ONE edge, or the
+        // exact-match reuse check breaks forever after.
+        const bySha = new Map();
+        for (const a of hashArtifacts(paths)) {
+            if (!bySha.has(a.sha256))
+                bySha.set(a.sha256, a);
+        }
+        const arts = [...bySha.values()];
+        if (arts.length === 0)
+            return undefined;
+        const wanted = new Set(arts.map((a) => a.sha256));
+        // 1. Resolve each artifact node by sha256 (find_by_prop).
+        const artIds = new Map(); // sha256 -> node id
+        for (const a of arts) {
+            const res = (await call("find_by_prop", {
+                label: "Artifact",
+                prop: "sha256",
+                value: a.sha256,
+                scope: "shared",
+            }));
+            const id = res?.nodes?.[0]?.id;
+            if (id)
+                artIds.set(a.sha256, id);
+        }
+        // 2. If every artifact exists, look for a PolicyVersion whose INCLUDES
+        //    set matches exactly (traverse inbound from any one artifact).
+        if (artIds.size === arts.length) {
+            const anyArt = [...artIds.values()][0];
+            const res = (await call("traverse", {
+                seed_id: anyArt,
+                max_hops: 2,
+                direction: "both",
+                edge_types: ["INCLUDES"],
+                scope: "shared",
+            }));
+            const nodes = res?.subgraph?.nodes ?? [];
+            const edges = res?.subgraph?.edges ?? [];
+            const shaById = new Map(nodes
+                .filter((n) => n.label === "Artifact")
+                .map((n) => [n.id, String(n.props?.sha256 ?? "")]));
+            for (const v of nodes.filter((n) => n.label === "PolicyVersion")) {
+                const included = edges
+                    .filter((e) => e.from === v.id)
+                    .map((e) => shaById.get(e.to))
+                    .filter(Boolean);
+                if (included.length === wanted.size &&
+                    included.every((s) => wanted.has(s))) {
+                    return v.id; // exact match — reuse, write nothing
+                }
+            }
+        }
+        // 3. Create what's missing in ONE batch: absent Artifacts, the new
+        //    PolicyVersion, INCLUDES edges. v1 intentionally does NOT maintain a
+        //    Harness node or an ACTIVE_POLICY edge pointing at "the current"
+        //    version — that would need the previous version's edge closed on
+        //    every rotation, which is out of scope here. Each episode instead
+        //    links directly to the PolicyVersion it used (see USED_POLICY in
+        //    recorder.ts), so "what was active when" is derivable per-episode
+        //    without a mutable singleton.
+        const cmds = [];
+        const refOf = new Map(); // sha256 -> "#N" or literal id
+        for (const a of arts) {
+            const existing = artIds.get(a.sha256);
+            if (existing) {
+                refOf.set(a.sha256, existing);
+            }
+            else {
+                refOf.set(a.sha256, `#${cmds.length}`);
+                cmds.push({
+                    op: "create_node",
+                    label: "Artifact",
+                    scope: "shared",
+                    props: { path: a.path, kind: a.kind, sha256: a.sha256, content: a.content },
+                });
+            }
+        }
+        const verRef = `#${cmds.length}`;
+        cmds.push({
+            op: "create_node",
+            label: "PolicyVersion",
+            scope: "shared",
+            props: { version: Date.now(), created_at: Date.now(), note: "recorder bootstrap" },
+        });
+        for (const a of arts) {
+            cmds.push({
+                op: "link",
+                from: verRef,
+                to: refOf.get(a.sha256),
+                type: "INCLUDES",
+            });
+        }
+        const res = (await call("submit_batch", { commands: cmds }));
+        const verIdx = cmds.findIndex((c) => c.label === "PolicyVersion");
+        return res?.ids?.[verIdx] ?? undefined;
+    }
+    catch (e) {
+        console.error(`topodb recorder: policy bootstrap failed: ${e.message}`);
+        return undefined;
+    }
+}

package/dist/recorder.js

@@ -0,0 +1,163 @@
+// src/recorder.ts — pure episode-recording core: no Pi imports, no I/O.
+// Everything here is deterministic and unit-tested; extension.ts supplies
+// the events and ships the output to topodb-mcp submit_batch.
+/** In-memory state of the currently open episode (one agent run). */
+export class EpisodeBuffer {
+    startedAt = 0;
+    turns = 0;
+    toolErrors = 0;
+    retrievals = [];
+    isOpen = false;
+    start(nowMs) {
+        this.startedAt = nowMs;
+        this.turns = 0;
+        this.toolErrors = 0;
+        this.retrievals = [];
+        this.isOpen = true;
+    }
+    addRetrieval(r) {
+        if (this.isOpen)
+            this.retrievals.push(r);
+    }
+    bumpTurns() {
+        if (this.isOpen)
+            this.turns++;
+    }
+    noteToolError() {
+        if (this.isOpen)
+            this.toolErrors++;
+    }
+    close() {
+        this.isOpen = false;
+    }
+    get open() {
+        return this.isOpen;
+    }
+}
+/** Message content -> plain text: strings pass through, block arrays
+ * contribute only their `type === "text"` blocks. Defensive: anything
+ * unrecognized contributes nothing. */
+export function extractText(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content
+        .map((b) => b && typeof b === "object" && b.type === "text"
+        ? String(b.text ?? "")
+        : "")
+        .join("");
+}
+/** Spec tokenization: lowercase alphanumeric runs of length >= 3, deduped,
+ * insertion order preserved. */
+export function tokenize(text) {
+    const out = new Set();
+    for (const m of text.toLowerCase().matchAll(/[a-z0-9]{3,}/g))
+        out.add(m[0]);
+    return out;
+}
+/** Spec USED rule: >= 50% of the memory's tokens appear in the text. A
+ * memory with no tokens is never "used". */
+export function isUsed(memContent, text) {
+    const mem = tokenize(memContent);
+    if (mem.size === 0)
+        return false;
+    const hay = tokenize(text);
+    let hits = 0;
+    for (const t of mem)
+        if (hay.has(t))
+            hits++;
+    return hits / mem.size >= 0.5;
+}
+function collect(node, i, score, out, contents) {
+    const id = node?.id;
+    if (typeof id !== "string")
+        return;
+    out.push({ id, rank: i, score });
+    const content = node?.props?.content;
+    if (typeof content === "string")
+        contents.set(id, content);
+}
+/** Map a `search_memories`/`traverse` tool result to a `RetrievalRecord` plus
+ * the memory contents it surfaced, or `undefined` when the tool isn't a
+ * retrieval call or the result doesn't match the expected wire shape (never
+ * throws — recording must never break the agent). Field names follow
+ * topodb-mcp's actual JSON (captured from the running server, not guessed):
+ * `search_memories` -> `{hits: [{node, score}]}`; `traverse` -> `{subgraph:
+ * {nodes, edges}}` where edges use `type` (not `ty`). */
+export function toRetrievalRecord(tool, args, result) {
+    const contents = new Map();
+    const returned = [];
+    if (tool === "search_memories") {
+        const hits = result?.hits;
+        if (!Array.isArray(hits))
+            return undefined;
+        hits.forEach((h, i) => {
+            const score = typeof h?.score === "number" ? h.score : 0;
+            collect(h?.node, i, score, returned, contents);
+        });
+        const query = typeof args.query === "string" ? args.query : "";
+        return { record: { query, at: Date.now(), channel: "text", returned }, contents };
+    }
+    if (tool === "traverse") {
+        const nodes = result?.subgraph?.nodes;
+        if (!Array.isArray(nodes))
+            return undefined;
+        nodes.forEach((n, i) => collect(n, i, 0, returned, contents));
+        const query = typeof args.seed_id === "string" ? args.seed_id : "";
+        return { record: { query, at: Date.now(), channel: "graph", returned }, contents };
+    }
+    return undefined;
+}
+/** Assemble the single atomic submit_batch command array for one episode.
+ * `used` maps retrieval index -> the set of memory ids judged used. */
+export function buildEpisodeBatch(args) {
+    const { buffer } = args;
+    const cmds = [
+        {
+            op: "create_node",
+            label: "Episode",
+            props: {
+                goal: args.goal,
+                strategy: "", // a reflection layer fills this later; never guess
+                outcome: args.outcome,
+                started_at: buffer.startedAt,
+                ended_at: args.endedAt,
+                turns: buffer.turns,
+                tokens: args.tokens,
+                confidence: 0.5,
+                failure: args.failure,
+            },
+        },
+    ];
+    buffer.retrievals.forEach((r, i) => {
+        const evRef = `#${cmds.length}`;
+        cmds.push({
+            op: "create_node",
+            label: "RetrievalEvent",
+            props: { query: r.query, at: r.at },
+        });
+        cmds.push({ op: "link", from: "#0", to: evRef, type: "ISSUED" });
+        for (const m of r.returned) {
+            cmds.push({
+                op: "link",
+                from: evRef,
+                to: m.id,
+                type: "RETURNED",
+                props: { rank: m.rank, score: m.score, channel: r.channel },
+            });
+        }
+        for (const id of args.used.get(i) ?? []) {
+            cmds.push({ op: "link", from: evRef, to: id, type: "USED" });
+        }
+    });
+    if (args.policyVersionId) {
+        cmds.push({
+            op: "link",
+            from: "#0",
+            to: args.policyVersionId,
+            type: "USED_POLICY",
+        });
+    }
+    return cmds;
+}

package/dist/server-handle.js

@@ -1,7 +1,25 @@
 // src/server-handle.ts
 import { createRequire } from "node:module";
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
 import { McpStdioClient } from "./mcp-client.js";
 const require = createRequire(import.meta.url);
+/** Recording defaults ON; `TOPODB_RECORD=0` disables it. */
+export function recordingEnabled(env) {
+    return env.TOPODB_RECORD !== "0";
+}
+/** `--spec` launch args pointing at the bundled episode IndexSpec, or `[]`
+ * when recording is disabled. `../spec/` is relative to the COMPILED
+ * `dist/server-handle.js` — `spec/` sits at the package root alongside
+ * `dist/`, so from `dist/` this resolves correctly both in-repo and once
+ * published (see package.json `files`). */
+export function episodeSpecArgs(env) {
+    if (!recordingEnabled(env))
+        return [];
+    const specPath = new URL("../spec/episode-index-spec.json", import.meta.url);
+    return ["--spec", fileURLToPath(specPath)];
+}
 export class TopodbServer {
     env;
     client;
@@ -12,15 +30,24 @@ export class TopodbServer {
     static resolveLauncher() {
         return require.resolve("@topodb/topodb-mcp/bin/topodb-mcp.js");
     }
-    args() {
-        const db = this.env.TOPODB_DB || ".topodb/memory.redb";
-        const scope = this.env.TOPODB_SCOPE || "shared";
-        return [TopodbServer.resolveLauncher(), "--db", db, "--scope", scope];
-    }
     async ensure() {
         if (this.client?.running)
             return this.client;
-        this.client = new McpStdioClient(this.args());
+        const db = this.env.TOPODB_DB || ".topodb/memory.redb";
+        const scope = this.env.TOPODB_SCOPE || "shared";
+        // topodb-mcp creates the db file on open but treats a missing parent
+        // directory as a startup error — and the default `.topodb/` won't exist in
+        // a fresh project. Create it so the server comes up on first use.
+        mkdirSync(dirname(db), { recursive: true });
+        const args = [
+            TopodbServer.resolveLauncher(),
+            "--db",
+            db,
+            "--scope",
+            scope,
+            ...episodeSpecArgs(this.env),
+        ];
+        this.client = new McpStdioClient(args);
         await this.client.start();
         return this.client;
     }

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@topodb/pi",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "Pi (pi.dev) extension: topodb agent memory via one install.",
   "type": "module",
   "license": "MIT OR Apache-2.0",
   "repository": { "type": "git", "url": "https://github.com/TopoDB/TopoDB" },
   "keywords": ["pi-package", "pi", "topodb", "memory", "mcp"],
   "pi": { "extensions": ["./dist/extension.js"] },
-  "files": ["dist/", "README.md"],
+  "files": ["dist/", "spec/", "README.md"],
   "engines": { "node": ">=22.19.0" },
   "scripts": {
     "build": "tsc",
@@ -15,7 +15,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@topodb/topodb-mcp": "0.0.3",
+    "@topodb/topodb-mcp": "0.0.5",
     "typebox": "1.1.38"
   },
   "devDependencies": {

package/spec/episode-index-spec.json

@@ -0,0 +1,14 @@
+{
+  "equality": [
+    { "label": "Entity", "prop": "name" },
+    { "label": "Artifact", "prop": "sha256" },
+    { "label": "PolicyVersion", "prop": "version" },
+    { "label": "Procedure", "prop": "name" },
+    { "label": "Episode", "prop": "outcome" }
+  ],
+  "text": [
+    { "label": "Memory", "prop": "content" },
+    { "label": "Lesson", "prop": "content" },
+    { "label": "Episode", "prop": "goal" }
+  ]
+}