@zenera/neo 1.1.11 → 1.1.14

package/dist/memory/store.js

@@ -0,0 +1,226 @@
+import { readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import { hostname } from 'node:os';
+import { join } from 'node:path';
+import { MemoryGraph } from "./graph.js";
+import { MemoryError } from "./types.js";
+import { VectorBlock } from "./vectors.js";
+// ---------------------------------------------------------------------------
+// What a memory is, on disk
+//
+// A directory, and the order it is written in is the whole crash story: the
+// manifest goes last, so a half-written memory has no manifest and reads as
+// empty rather than as a graph that quietly lost its vectors. Every file lands
+// through a temporary name and a rename, which on a local filesystem is atomic,
+// so a reader never sees half a graph.
+//
+// The lock is advisory and per directory. Memory is project-scoped, and two
+// runs of the same project writing the same graph would interleave commits and
+// lose edges — but a lock whose process is gone is stale by definition and gets
+// taken, because a crashed run must not make a project permanently unusable.
+//
+// Vectors live beside the graph rather than inside it. `graph.json` is parsed
+// whole on open and a few thousand embeddings inflate to megabytes of JSON
+// numbers; the same data as raw little-endian float32 is a quarter the size and
+// needs no parsing at all.
+// ---------------------------------------------------------------------------
+export const MEMORY_VERSION = 1;
+export const GRAPH_FILE = 'graph.json';
+export const VECTORS_FILE = 'vectors.f32';
+export const VECTOR_IDS_FILE = 'vectors.json';
+export const MANIFEST_FILE = 'manifest.json';
+export const FILES_DIR = 'files';
+export const LOCK_FILE = '.lock';
+export class MemoryStore {
+    dir;
+    graph;
+    #embedding;
+    #vectors;
+    #locked = false;
+    constructor(dir, graph, vectors, embedding) {
+        this.dir = dir;
+        this.graph = graph;
+        this.#vectors = vectors;
+        this.#embedding = embedding;
+    }
+    static async open(dir, opts = {}) {
+        await mkdir(join(dir, FILES_DIR), { recursive: true });
+        const manifest = await readManifest(dir);
+        if (manifest?.embedding && opts.embedding) {
+            const a = manifest.embedding;
+            const b = opts.embedding;
+            if (a.model !== b.model || a.dimensions !== b.dimensions) {
+                throw new MemoryError(`this memory was embedded with ${a.model} (${a.dimensions}d), ` +
+                    `but ${b.model} (${b.dimensions}d) is configured`, 're-embed it with `zen memory reembed`, or restore the original model');
+            }
+        }
+        const graph = manifest
+            ? MemoryGraph.from(JSON.parse(await readFile(join(dir, GRAPH_FILE), 'utf8')))
+            : new MemoryGraph();
+        const embedding = opts.embedding ?? manifest?.embedding;
+        const vectors = manifest && embedding ? await readVectors(dir, embedding) : undefined;
+        const store = new MemoryStore(dir, graph, vectors, embedding);
+        if (opts.lock !== false) {
+            store.#claim();
+        }
+        return store;
+    }
+    get embedding() {
+        return this.#embedding;
+    }
+    /**
+     * Records the vectoriser actually in use. A width is only knowable from a
+     * real response, so a memory embedded for the first time learns it here
+     * rather than making the caller declare a number it would have to keep in
+     * step with the model.
+     */
+    adopt(embedding) {
+        const held = this.#embedding;
+        if (held) {
+            if (held.model !== embedding.model || held.dimensions !== embedding.dimensions) {
+                throw new MemoryError(`this memory was embedded with ${held.model} (${held.dimensions}d), ` +
+                    `but ${embedding.model} (${embedding.dimensions}d) is configured`, 're-embed it with `zen memory reembed`, or restore the original model');
+            }
+            return;
+        }
+        this.#embedding = embedding;
+    }
+    /** Absent when no embedder is configured; recall then falls back to term overlap. */
+    get vectors() {
+        if (!this.#vectors && this.#embedding) {
+            this.#vectors = new VectorBlock(this.#embedding.dimensions);
+        }
+        return this.#vectors;
+    }
+    filePath(name) {
+        return join(this.dir, FILES_DIR, name);
+    }
+    /** Manifest last: a torn commit reads as no memory, never as a partial one. */
+    async commit() {
+        await atomic(join(this.dir, GRAPH_FILE), JSON.stringify(this.graph.export()));
+        const vectors = this.#vectors;
+        if (vectors) {
+            const raw = vectors.bytes();
+            await atomic(join(this.dir, VECTORS_FILE), new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength));
+            await atomic(join(this.dir, VECTOR_IDS_FILE), JSON.stringify({ dims: vectors.dims, ids: vectors.ids }));
+        }
+        const manifest = {
+            version: MEMORY_VERSION,
+            ...(this.#embedding ? { embedding: this.#embedding } : {}),
+            nodes: this.graph.order,
+            edges: this.graph.size,
+            updatedAt: new Date().toISOString(),
+        };
+        await atomic(join(this.dir, MANIFEST_FILE), `${JSON.stringify(manifest, null, 4)}\n`);
+    }
+    release() {
+        if (!this.#locked) {
+            return;
+        }
+        this.#locked = false;
+        try {
+            unlinkSync(join(this.dir, LOCK_FILE));
+        }
+        catch {
+            // Already gone, or taken over as stale: neither is worth failing a run for.
+        }
+    }
+    /**
+     * `wx` makes the create and the check one operation, so two runs racing for
+     * the same project cannot both win.
+     */
+    #claim() {
+        const path = join(this.dir, LOCK_FILE);
+        const lock = {
+            pid: process.pid,
+            host: hostname(),
+            startedAt: new Date().toISOString(),
+        };
+        const body = `${JSON.stringify(lock, null, 4)}\n`;
+        try {
+            writeFileSync(path, body, { flag: 'wx' });
+            this.#locked = true;
+            return;
+        }
+        catch (err) {
+            if (err.code !== 'EEXIST') {
+                throw err;
+            }
+        }
+        const held = readLock(path);
+        if (held && held.host === hostname() && alive(held.pid)) {
+            throw new MemoryError(`this memory is in use (pid ${held.pid}, since ${held.startedAt})`, 'wait for that run to finish, or remove .lock if it crashed');
+        }
+        writeFileSync(path, body);
+        this.#locked = true;
+    }
+}
+async function readManifest(dir) {
+    try {
+        const manifest = JSON.parse(await readFile(join(dir, MANIFEST_FILE), 'utf8'));
+        if (manifest.version !== MEMORY_VERSION) {
+            throw new MemoryError(`this memory is version ${manifest.version}, but this build reads version ${MEMORY_VERSION}`, 'export it with the older build, or start a new memory');
+        }
+        return manifest;
+    }
+    catch (err) {
+        if (err instanceof MemoryError) {
+            throw err;
+        }
+        return undefined;
+    }
+}
+async function readVectors(dir, embedding) {
+    let ids;
+    let dims;
+    try {
+        const meta = JSON.parse(await readFile(join(dir, VECTOR_IDS_FILE), 'utf8'));
+        ids = meta.ids;
+        dims = meta.dims;
+    }
+    catch {
+        return new VectorBlock(embedding.dimensions);
+    }
+    if (dims !== embedding.dimensions) {
+        return new VectorBlock(embedding.dimensions);
+    }
+    const raw = await readFile(join(dir, VECTORS_FILE));
+    // Copied rather than viewed: a Buffer from the pool is not guaranteed to sit
+    // on a 4-byte boundary, and Float32Array over an unaligned buffer throws.
+    const data = new Float32Array(Math.floor(raw.byteLength / 4));
+    Buffer.from(data.buffer, data.byteOffset, data.byteLength).set(raw);
+    return VectorBlock.load(dims, ids, data);
+}
+async function atomic(path, body) {
+    const tmp = `${path}.tmp`;
+    await writeFile(tmp, body);
+    try {
+        await rename(tmp, path);
+    }
+    catch (err) {
+        await rm(tmp, { force: true });
+        throw err;
+    }
+}
+function readLock(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `kill(pid, 0)` sends no signal and only asks whether the process exists.
+ * EPERM means it exists and belongs to someone else, which still counts.
+ */
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/memory/tools.d.ts

@@ -0,0 +1,11 @@
+import { type AnyTool } from '../types.ts';
+import type { MemoryIndex } from './index.ts';
+import { type ResolvedMemoryBinding } from './types.ts';
+/** Deterministic, so a replayed call is the same op and not a second one. */
+export declare function memoryOpId(runId: string, callId: string): string;
+export interface MemoryToolsOptions {
+    index: MemoryIndex;
+    binding: ResolvedMemoryBinding;
+}
+export declare function memoryTools<TCtx>(opts: MemoryToolsOptions): AnyTool<TCtx>[];
+//# sourceMappingURL=tools.d.ts.map

package/dist/memory/tools.js

@@ -0,0 +1,337 @@
+/**
+ * The four tools an agent gets. Deliberately four and not seven: `search` and
+ * `load` are split because finding is not reading — search returns stitched
+ * context the model never asked for, and counting that as use would poison
+ * recency. `commit` is one transaction because a remembered thing is a
+ * subgraph, and building it with three calls leaves the graph half-formed if
+ * the model stops early. `forget` is separate because deleting rarely happens
+ * in the same breath as creating, and a `forget` buried inside a truncated
+ * commit preview would be both irreversible and invisible.
+ *
+ * `audience` reaches the schema only when the binding grants more than one
+ * label, and `sees` never does: an agent that could name its own audience
+ * could read another agent's slice just by asking for it.
+ */
+import { hash } from "../payload.js";
+import { MEMORY_COMMIT_TOOL, MEMORY_FORGET_TOOL, MEMORY_LOAD_TOOL, MEMORY_SEARCH_TOOL, tool, withEffects, } from "../types.js";
+import { renderRecollection } from "./render.js";
+import { canForget, canWrite, KIND_HELP, MemoryError, RELATION_HELP, } from "./types.js";
+/** Deterministic, so a replayed call is the same op and not a second one. */
+export function memoryOpId(runId, callId) {
+    return hash(`${runId}\u0000${callId}`);
+}
+export function memoryTools(opts) {
+    const { index, binding } = opts;
+    const kinds = [...index.kinds];
+    const relations = [...index.relations];
+    const kindHelp = kinds
+        .map((k) => `${k}: ${KIND_HELP[k] ?? ''}`)
+        .join('; ');
+    const relationHelp = relations
+        .map((r) => `${r}: ${RELATION_HELP[r] ?? ''}`)
+        .join('; ');
+    const tools = [
+        tool({
+            name: MEMORY_SEARCH_TOOL,
+            description: 'Search memory for what was learned on earlier runs. Returns a diagram of ' +
+                'the matching subgraph plus a legend of ids; read a node with ' +
+                `${MEMORY_LOAD_TOOL}.`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    query: { type: 'string', description: 'what you are trying to recall' },
+                    kinds: {
+                        type: 'array',
+                        items: { type: 'string', enum: kinds },
+                        description: 'restrict to these kinds',
+                    },
+                    limit: { type: 'integer', description: 'how many matches to seed from' },
+                    max_hops: { type: 'integer', description: 'how far to follow links' },
+                    max_nodes: { type: 'integer', description: 'cap on the returned subgraph' },
+                    newer_than: {
+                        type: 'string',
+                        description: 'ISO timestamp; ignore anything older',
+                    },
+                },
+                required: ['query'],
+                additionalProperties: false,
+            },
+            execute: async (args, tc) => {
+                const query = {
+                    text: args.query,
+                    kinds: args.kinds,
+                    limit: args.limit,
+                    maxHops: args.max_hops,
+                    maxNodes: args.max_nodes,
+                    newerThan: args.newer_than,
+                };
+                const rec = await index.search(query, binding.sees);
+                const content = renderRecollection(rec);
+                if (!content) {
+                    return 'no memory of this';
+                }
+                return withEffects(content, {
+                    kind: 'memory_op',
+                    spec: {
+                        kind: 'recall',
+                        query,
+                        seeds: [...rec.seeds],
+                        nodes: rec.nodes.map((n) => ({
+                            id: n.node.id,
+                            kind: n.node.kind,
+                            score: n.score,
+                        })),
+                        edges: [...rec.edges],
+                        content,
+                    },
+                });
+            },
+        }),
+        tool({
+            name: MEMORY_LOAD_TOOL,
+            description: 'Read memories in full by id, using the ids from a ' +
+                `${MEMORY_SEARCH_TOOL} legend. A remembered file comes back inline when it ` +
+                'is small, otherwise as a path under /memory you can open or run.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    ids: { type: 'array', items: { type: 'string' }, minItems: 1 },
+                },
+                required: ['ids'],
+                additionalProperties: false,
+            },
+            execute: async (args, tc) => {
+                const opId = memoryOpId(tc.state.runId, tc.callId);
+                const loaded = await index.load(args.ids, binding.sees, tc.services.clock);
+                // Recorded because this is the only path that puts a whole body
+                // in the context: a recall block carries a clipped line.
+                const spec = {
+                    kind: 'op',
+                    op: 'load',
+                    opId,
+                    nodes: loaded.map((l) => ({
+                        id: l.node.id,
+                        kind: l.node.kind,
+                        revision: l.node.revision,
+                    })),
+                    edges: [],
+                    files: loaded.filter((l) => l.content !== undefined).length,
+                };
+                return withEffects(loaded.map((l) => ({
+                    id: l.node.id,
+                    kind: l.node.kind,
+                    text: l.node.text,
+                    created: l.node.createdAt,
+                    revision: l.node.revision,
+                    metadata: l.node.metadata,
+                    file: l.node.file
+                        ? { path: l.node.file.path, bytes: l.node.file.bytes }
+                        : undefined,
+                    content: l.content,
+                })), { kind: 'memory_op', spec });
+            },
+        }),
+    ];
+    if (!canWrite(binding)) {
+        return tools;
+    }
+    const labels = binding.writes;
+    const audience = labels.length > 1
+        ? {
+            audience: {
+                type: 'array',
+                items: { type: 'string', enum: labels },
+                description: 'who may later see this node; defaults to everyone',
+            },
+        }
+        : {};
+    tools.push(tool({
+        name: MEMORY_COMMIT_TOOL,
+        description: 'Remember something as a small graph, in one call: the nodes and the links ' +
+            'between them. Link a new node to an existing one by using its id. A node ' +
+            'that already exists is folded into it, and its id comes back under your ref.',
+        parameters: {
+            type: 'object',
+            properties: {
+                nodes: {
+                    type: 'array',
+                    minItems: 1,
+                    items: {
+                        type: 'object',
+                        properties: {
+                            ref: {
+                                type: 'string',
+                                description: 'a short name for a new node, used by edges in this call',
+                            },
+                            id: {
+                                type: 'string',
+                                description: 'to change an existing node instead of adding one',
+                            },
+                            kind: { type: 'string', enum: kinds, description: kindHelp },
+                            text: {
+                                type: 'string',
+                                description: 'what to remember, in your own words',
+                            },
+                            file: {
+                                type: 'string',
+                                description: 'path of a file to keep a copy of, for kind "file"',
+                            },
+                            ...audience,
+                            expected_revision: {
+                                type: 'integer',
+                                description: 'refuse the change if the node moved on',
+                            },
+                        },
+                        additionalProperties: false,
+                    },
+                },
+                edges: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            from: {
+                                type: 'string',
+                                description: 'a ref from this call, or an id',
+                            },
+                            to: {
+                                type: 'string',
+                                description: 'a ref from this call, or an id',
+                            },
+                            relation: {
+                                type: 'string',
+                                enum: relations,
+                                description: relationHelp,
+                            },
+                        },
+                        required: ['from', 'to', 'relation'],
+                        additionalProperties: false,
+                    },
+                },
+            },
+            required: ['nodes'],
+            additionalProperties: false,
+        },
+        execute: async (args, tc) => {
+            const opId = memoryOpId(tc.state.runId, tc.callId);
+            let nodes;
+            try {
+                nodes = args.nodes.map((n) => ({
+                    ref: n.ref,
+                    id: n.id,
+                    kind: n.kind,
+                    text: n.text,
+                    audience: n.audience,
+                    expectedRevision: n.expected_revision,
+                    file: n.file === undefined ? undefined : { source: source(tc, n.file) },
+                }));
+            }
+            catch (err) {
+                return refusal(err);
+            }
+            const edges = (args.edges ?? []).map((e) => ({
+                from: e.from,
+                to: e.to,
+                relation: e.relation,
+            }));
+            let res;
+            try {
+                res = await index.commit({ nodes, edges }, { writes: labels, sees: binding.sees, clock: tc.services.clock });
+            }
+            catch (err) {
+                return refusal(err);
+            }
+            // The TUI shows a truncated preview, so the counts go first.
+            const parts = [`${MEMORY_COMMIT_TOOL} ${res.created + res.updated} nodes`];
+            if (res.edges) {
+                parts.push(`${res.edges} edges`);
+            }
+            if (res.files) {
+                parts.push(`${res.files} files`);
+            }
+            if (res.merged) {
+                parts.push(`${res.merged} already known`);
+            }
+            const spec = {
+                kind: 'op',
+                op: 'commit',
+                opId,
+                nodes: changed(index, res.ids),
+                edges: [],
+                files: res.files,
+            };
+            return withEffects({ summary: parts.join(' '), ids: res.ids }, { kind: 'memory_op', spec });
+        },
+    }));
+    if (!canForget(binding)) {
+        return tools;
+    }
+    tools.push(tool({
+        name: MEMORY_FORGET_TOOL,
+        description: 'Delete memories by id, with the files they hold. Use this only for what is ' +
+            'wrong or no longer allowed to be kept; to correct something, prefer a new ' +
+            `node linked with SUPERSEDES in ${MEMORY_COMMIT_TOOL}.`,
+        parameters: {
+            type: 'object',
+            properties: {
+                ids: { type: 'array', items: { type: 'string' }, minItems: 1 },
+                reason: { type: 'string', description: 'why this is being removed' },
+            },
+            required: ['ids'],
+            additionalProperties: false,
+        },
+        execute: async (args, tc) => {
+            const opId = memoryOpId(tc.state.runId, tc.callId);
+            const before = args.ids.flatMap((id) => {
+                const node = index.graph.get(id);
+                return node ? [{ id, kind: node.kind, revision: node.revision }] : [];
+            });
+            // Taken before the delete: afterwards nothing can say what this
+            // node hung from, and a report would draw it floating.
+            const links = new Map(args.ids
+                .flatMap((id) => index.graph.neighbors(id, binding.sees))
+                .map((e) => [`${e.source}|${e.relation}|${e.target}`, e]));
+            try {
+                await index.forget(args.ids, binding.sees);
+            }
+            catch (err) {
+                return refusal(err);
+            }
+            return withEffects(`${MEMORY_FORGET_TOOL} ${before.length} nodes`, {
+                kind: 'memory_op',
+                spec: {
+                    kind: 'op',
+                    op: 'forget',
+                    opId,
+                    nodes: before,
+                    edges: [...links.values()],
+                    files: 0,
+                },
+            });
+        },
+    }));
+    return tools;
+}
+/**
+ * A refusal is a result, not a throw: the model can read it and fix the call,
+ * whereas an exception ends the turn.
+ */
+function refusal(err) {
+    if (err instanceof MemoryError) {
+        return { error: err.message, hint: err.hint };
+    }
+    throw err;
+}
+function source(tc, path) {
+    if (!tc.services.resolveFile) {
+        throw new MemoryError('this agent cannot remember files', 'commit the node without `file`, or give the agent a workspace');
+    }
+    return tc.services.resolveFile(path);
+}
+function changed(index, ids) {
+    return Object.values(ids).flatMap((id) => {
+        const node = index.graph.get(id);
+        return node ? [{ id, kind: node.kind, revision: node.revision }] : [];
+    });
+}
+//# sourceMappingURL=tools.js.map