@zenera/neo 1.1.11 → 1.1.12

package/dist/memory/index.d.ts

@@ -0,0 +1,93 @@
+import type { Embedder } from '../embedding.ts';
+import type { IdClock } from '../ids.ts';
+import type { MemoryGraph } from './graph.ts';
+import type { MemoryStore } from './store.ts';
+import { type MemoryNode, type MemoryQuery, type Recollection } from './types.ts';
+/** Past this a `load` returns the path and lets the agent open it itself. */
+export declare const INLINE_FILE_BYTES: number;
+/**
+ * Cosine at or above this is one memory written twice, not two memories. Set
+ * high on purpose: merging two things that only resembled each other loses a
+ * memory outright, while missing a merge only wastes a node.
+ */
+export declare const DUPLICATE_SCORE = 0.97;
+export interface MemoryIndexOptions {
+    store: MemoryStore;
+    /** absent ranks by term overlap and stores no vectors */
+    embedder?: Embedder;
+    /** the project's vocabulary, when it extends the built-in one */
+    kinds?: readonly string[];
+    relations?: readonly string[];
+}
+export interface CommitNode {
+    /** names this node within the transaction; for a node being created */
+    ref?: string;
+    /** an existing node to update; mutually exclusive with `ref` */
+    id?: string;
+    kind?: string;
+    text?: string;
+    audience?: string[];
+    metadata?: Record<string, unknown>;
+    /** absolute host path of a file to copy in, already resolved by the caller */
+    file?: {
+        source: string;
+    };
+    expectedRevision?: number;
+}
+export interface CommitEdge {
+    from: string;
+    to: string;
+    relation: string;
+}
+export interface MemoryTransaction {
+    nodes?: CommitNode[];
+    edges?: CommitEdge[];
+}
+export interface CommitOptions {
+    /** audience labels this agent may assign */
+    writes: readonly string[];
+    /** labels it may read; a duplicate is only a duplicate if it can see it */
+    sees: readonly string[];
+    clock: IdClock;
+}
+export interface CommitResult {
+    /** local ref (or id, for an update) to the id it resolved to */
+    ids: Record<string, string>;
+    created: number;
+    updated: number;
+    edges: number;
+    files: number;
+    /** new nodes that turned out to already exist, and were folded into them */
+    merged: number;
+}
+export interface LoadedNode {
+    node: MemoryNode;
+    /** the file's bytes, when it was small enough to inline */
+    content?: string;
+}
+export declare class MemoryIndex {
+    #private;
+    readonly store: MemoryStore;
+    constructor(opts: MemoryIndexOptions);
+    get graph(): MemoryGraph;
+    get kinds(): readonly string[];
+    get relations(): readonly string[];
+    /** Read-only: a search returns nodes the model has not read, so nothing is touched. */
+    search(query: MemoryQuery, sees: readonly string[]): Promise<Recollection>;
+    /**
+     * Listed rather than ranked, and deliberately not routed through `recall`:
+     * ranking decays by `lastUsedAt`, only `memory_load` bumps that, and a
+     * preference is injected rather than loaded — so it would age out of its
+     * own list. Ordered so the rendered block is byte-stable between runs,
+     * because an unstable one would invalidate the prompt prefix for nothing.
+     */
+    preferences(sees: readonly string[]): MemoryNode[];
+    /**
+     * The only path that counts as use — recency decay is meant to track what
+     * an agent actually read, and a search hands back context it never opened.
+     */
+    load(ids: readonly string[], sees: readonly string[], clock: IdClock): Promise<LoadedNode[]>;
+    commit(tx: MemoryTransaction, opts: CommitOptions): Promise<CommitResult>;
+    forget(ids: readonly string[], sees: readonly string[]): Promise<MemoryNode[]>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/memory/index.js

@@ -0,0 +1,357 @@
+import { readFile } from 'node:fs/promises';
+import { forgetFile, hostPath, rememberFile } from "./files.js";
+import { recall } from "./recall.js";
+import { ALL_AGENTS, MEMORY_KINDS, MEMORY_RELATIONS, MemoryError, PREFERENCE_KIND, visible, } from "./types.js";
+// ---------------------------------------------------------------------------
+// The memory, as the engine uses it
+//
+// `MemoryStore` knows how to persist a graph; this knows what the operations
+// mean. It owns the two things that must not drift: the vector for a node's
+// text, and the bytes of a remembered file.
+//
+// `commit` is one transaction. The model names nodes it is creating by a local
+// `ref` — it cannot know an id that has not been minted — and edges join refs
+// and existing ids indiscriminately, so a whole subgraph arrives in one call.
+// Everything is validated, and every source file stat-ed, before the first
+// mutation, on the same principle as `runPatch` in tools/workspace.ts: a
+// commit that fails must leave no half-built graph behind.
+// ---------------------------------------------------------------------------
+/** Past this a `load` returns the path and lets the agent open it itself. */
+export const INLINE_FILE_BYTES = 32 * 1024;
+/**
+ * Cosine at or above this is one memory written twice, not two memories. Set
+ * high on purpose: merging two things that only resembled each other loses a
+ * memory outright, while missing a merge only wastes a node.
+ */
+export const DUPLICATE_SCORE = 0.97;
+export class MemoryIndex {
+    store;
+    #embedder;
+    #kinds;
+    #relations;
+    constructor(opts) {
+        this.store = opts.store;
+        this.#embedder = opts.embedder;
+        this.#kinds = new Set([...(opts.kinds ?? MEMORY_KINDS), PREFERENCE_KIND]);
+        this.#relations = new Set(opts.relations ?? MEMORY_RELATIONS);
+    }
+    get graph() {
+        return this.store.graph;
+    }
+    get kinds() {
+        return [...this.#kinds];
+    }
+    get relations() {
+        return [...this.#relations];
+    }
+    /** Read-only: a search returns nodes the model has not read, so nothing is touched. */
+    async search(query, sees) {
+        const vector = query.text ? await this.#embed([query.text], 'query') : undefined;
+        return recall({
+            graph: this.graph,
+            query,
+            sees,
+            vector: vector?.[0],
+            vectors: this.store.vectors,
+        });
+    }
+    /**
+     * Listed rather than ranked, and deliberately not routed through `recall`:
+     * ranking decays by `lastUsedAt`, only `memory_load` bumps that, and a
+     * preference is injected rather than loaded — so it would age out of its
+     * own list. Ordered so the rendered block is byte-stable between runs,
+     * because an unstable one would invalidate the prompt prefix for nothing.
+     */
+    preferences(sees) {
+        const stale = this.graph.superseded(sees);
+        return this.graph
+            .nodes(sees)
+            .filter((n) => n.kind === PREFERENCE_KIND && !stale.has(n.id))
+            .sort((a, b) => a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id));
+    }
+    /**
+     * The only path that counts as use — recency decay is meant to track what
+     * an agent actually read, and a search hands back context it never opened.
+     */
+    async load(ids, sees, clock) {
+        const at = clock.now();
+        const out = [];
+        for (const id of ids) {
+            const node = this.graph.get(id);
+            // Indistinguishable from "no such node" on purpose: a refusal that
+            // said "you may not see this" would confirm the node exists.
+            if (!node || !visible(node, sees)) {
+                throw new MemoryError(`no memory ${id}`, 'use an id from the legend of a memory_search result');
+            }
+            const touched = this.graph.touch(id, at) ?? node;
+            const file = touched.file;
+            out.push({
+                node: touched,
+                content: file && file.bytes <= INLINE_FILE_BYTES
+                    ? await readFile(hostPath(this.store.dir, file), 'utf8')
+                    : undefined,
+            });
+        }
+        await this.store.commit();
+        return out;
+    }
+    async commit(tx, opts) {
+        const nodes = tx.nodes ?? [];
+        const edges = tx.edges ?? [];
+        if (!nodes.length && !edges.length) {
+            throw new MemoryError('a commit needs at least one node or edge');
+        }
+        const plan = this.#plan(nodes, edges, opts);
+        const vectors = await this.#embed(plan.embed.map((e) => e.text), 'document');
+        // Keyed before anything is dropped: `#merge` reshapes `plan.embed`, and
+        // the vectors only line up with it positionally until it does.
+        const vector = new Map(plan.embed.map((e, i) => [e.id, vectors?.[i]]));
+        const merged = this.#merge(plan, vector, opts.sees);
+        // Files first, so a failed copy can be undone before the graph knows.
+        const written = [];
+        try {
+            for (const item of plan.files) {
+                const file = await rememberFile(this.store.dir, item.source, item.id);
+                written.push(file);
+                item.target.file = file;
+            }
+        }
+        catch (err) {
+            await Promise.all(written.map((f) => forgetFile(this.store.dir, f)));
+            throw err;
+        }
+        const at = opts.clock.now();
+        let created = 0;
+        let updated = 0;
+        for (const item of plan.nodes) {
+            if (item.existing) {
+                this.graph.update(item.id, item.patch, at);
+                updated++;
+            }
+            else {
+                this.graph.add(item.draft, at);
+                created++;
+            }
+        }
+        for (const edge of plan.edges) {
+            this.graph.link(edge.source, edge.target, edge.relation, at);
+        }
+        for (const e of plan.embed) {
+            const v = vector.get(e.id);
+            if (v) {
+                this.store.vectors?.set(e.id, v);
+            }
+        }
+        await this.store.commit();
+        return {
+            ids: plan.ids,
+            created,
+            updated,
+            edges: plan.edges.length,
+            files: plan.files.length,
+            merged,
+        };
+    }
+    /**
+     * A second run that learns the same thing should join the graph, not fork
+     * it. Nothing tells an agent the id of a node it is about to write, so two
+     * runs of one job commit two identical tasks and split every edge between
+     * them — which is what makes a five-seed recall return two memories.
+     *
+     * Detected here rather than in `#plan` because the vectors that answer it
+     * have just been paid for, so the check is a dot product over what is
+     * already in memory and costs no extra call. A merged node keeps its
+     * original id, and every edge and ref in this commit is re-pointed at it,
+     * so the caller is told the canonical id and can link to it next time.
+     *
+     * File-bearing nodes never merge: their text is a summary of the artifact,
+     * and two summaries can read alike while the bytes underneath differ.
+     */
+    #merge(plan, vector, sees) {
+        const minted = new Set(plan.nodes.map((n) => n.id));
+        const carries = new Set(plan.files.map((f) => f.id));
+        const stale = this.graph.superseded(sees);
+        const twins = new Map();
+        for (const item of plan.nodes) {
+            if (item.existing || carries.has(item.id)) {
+                continue;
+            }
+            const { kind, text } = item.draft;
+            const eligible = (id) => {
+                if (minted.has(id) || stale.has(id)) {
+                    return false;
+                }
+                const node = this.graph.get(id);
+                return !!node && node.kind === kind && !node.file && visible(node, sees);
+            };
+            const mine = vector.get(item.id);
+            const twin = mine
+                ? this.store.vectors
+                    ?.topK(mine, 1, eligible)
+                    .find((h) => h.score >= DUPLICATE_SCORE)?.id
+                : this.graph
+                    .nodes(sees)
+                    .find((n) => eligible(n.id) && flatten(n.text) === flatten(text))?.id;
+            if (twin) {
+                twins.set(item.id, twin);
+            }
+        }
+        if (!twins.size) {
+            return 0;
+        }
+        const to = (id) => twins.get(id) ?? id;
+        plan.nodes = plan.nodes.filter((n) => !twins.has(n.id));
+        plan.embed = plan.embed.filter((e) => !twins.has(e.id));
+        for (const [ref, id] of Object.entries(plan.ids)) {
+            plan.ids[ref] = to(id);
+        }
+        // A self-edge is what two refs collapsing onto one node leaves behind.
+        plan.edges = plan.edges
+            .map((e) => ({ ...e, source: to(e.source), target: to(e.target) }))
+            .filter((e) => e.source !== e.target);
+        return twins.size;
+    }
+    async forget(ids, sees) {
+        const doomed = ids.map((id) => {
+            const node = this.graph.get(id);
+            if (!node || !visible(node, sees)) {
+                throw new MemoryError(`no memory ${id}`);
+            }
+            return node;
+        });
+        for (const node of doomed) {
+            this.graph.forget(node.id);
+            this.store.vectors?.remove(node.id);
+            if (node.file) {
+                await forgetFile(this.store.dir, node.file);
+            }
+        }
+        await this.store.commit();
+        return doomed;
+    }
+    /**
+     * Everything that can be refused, refused here — before a file is copied or
+     * a node is added, so a rejected commit leaves the graph exactly as it was.
+     */
+    #plan(nodes, edges, opts) {
+        const ids = {};
+        const planned = [];
+        const files = [];
+        const embed = [];
+        for (const [i, node] of nodes.entries()) {
+            const where = node.ref ?? node.id ?? `nodes[${i}]`;
+            if ((node.ref === undefined) === (node.id === undefined)) {
+                throw new MemoryError(`${where}: give either a ref (to create) or an id (to update), not both`);
+            }
+            const audience = this.#audience(node, opts.writes, where);
+            if (node.id) {
+                const existing = this.graph.get(node.id);
+                if (!existing) {
+                    throw new MemoryError(`${where}: no such memory`);
+                }
+                if (node.kind !== undefined) {
+                    this.#kind(node.kind, where);
+                }
+                ids[node.id] = node.id;
+                const patch = {
+                    ...(node.kind !== undefined ? { kind: node.kind } : {}),
+                    ...(node.text !== undefined ? { text: node.text } : {}),
+                    ...(audience ? { audience } : {}),
+                    ...(node.metadata !== undefined ? { metadata: node.metadata } : {}),
+                    ...(node.expectedRevision !== undefined
+                        ? { expectedRevision: node.expectedRevision }
+                        : {}),
+                };
+                planned.push({ existing: true, id: node.id, patch });
+                // Only a text change invalidates the vector; re-embedding on an
+                // audience edit would burn a call to store the same numbers.
+                if (node.text !== undefined && node.text !== existing.text) {
+                    embed.push({ id: node.id, text: node.text });
+                }
+                if (node.file) {
+                    files.push({ id: node.id, source: node.file.source, target: patch });
+                }
+                continue;
+            }
+            const ref = node.ref;
+            if (ref in ids) {
+                throw new MemoryError(`${where}: two nodes share the ref "${ref}"`);
+            }
+            if (node.text === undefined || !node.text.trim()) {
+                throw new MemoryError(`${where}: a new node needs text`, 'text is the only thing a search can match, so it has to describe the node');
+            }
+            const kind = this.#kind(node.kind, where);
+            const id = opts.clock.newId();
+            ids[ref] = id;
+            const draft = {
+                id,
+                kind,
+                text: node.text,
+                audience: audience ?? [ALL_AGENTS],
+                ...(node.metadata !== undefined ? { metadata: node.metadata } : {}),
+            };
+            planned.push({ existing: false, id, draft });
+            embed.push({ id, text: node.text });
+            if (node.file) {
+                files.push({ id, source: node.file.source, target: draft });
+            }
+        }
+        const resolved = edges.map((edge, i) => {
+            const where = `edges[${i}]`;
+            const relation = edge.relation;
+            if (!this.#relations.has(relation)) {
+                throw new MemoryError(`${where}: unknown relation "${relation}"`, `use one of ${[...this.#relations].join(', ')}`);
+            }
+            return {
+                source: this.#endpoint(edge.from, ids, where),
+                target: this.#endpoint(edge.to, ids, where),
+                relation: relation,
+            };
+        });
+        return { ids, nodes: planned, edges: resolved, files, embed };
+    }
+    #endpoint(name, ids, where) {
+        const id = ids[name];
+        if (id) {
+            return id;
+        }
+        if (this.graph.has(name)) {
+            return name;
+        }
+        throw new MemoryError(`${where}: "${name}" is neither a ref in this commit nor an existing memory id`);
+    }
+    #kind(kind, where) {
+        if (kind === undefined) {
+            throw new MemoryError(`${where}: a new node needs a kind`);
+        }
+        if (!this.#kinds.has(kind)) {
+            throw new MemoryError(`${where}: unknown kind "${kind}"`, `use one of ${[...this.#kinds].join(', ')}`);
+        }
+        return kind;
+    }
+    /** An agent can only label a node with something it was granted. */
+    #audience(node, writes, where) {
+        if (!node.audience?.length) {
+            return undefined;
+        }
+        for (const label of node.audience) {
+            if (!writes.includes(label)) {
+                throw new MemoryError(`${where}: not allowed to label a memory "${label}"`, `this agent may write ${writes.join(', ') || 'nothing'}`);
+            }
+        }
+        return [...node.audience];
+    }
+    async #embed(texts, taskType) {
+        if (!this.#embedder || !texts.length) {
+            return undefined;
+        }
+        const res = await this.#embedder.embed({ input: texts, taskType });
+        this.store.adopt({ model: this.#embedder.id, dimensions: res.dimensions });
+        return res.vectors;
+    }
+}
+/** Whitespace and case are not what makes two memories different. */
+function flatten(text) {
+    return text.replace(/\s+/g, ' ').trim().toLowerCase();
+}
+//# sourceMappingURL=index.js.map

package/dist/memory/instructions.d.ts

@@ -0,0 +1,15 @@
+import { type MemoryNode, type ResolvedMemoryBinding } from './types.ts';
+export declare const PREFERENCES_TAG = "memory-preferences";
+/**
+ * Deliberately not a function of `sees`: those labels come from the run's
+ * context, and naming them here would rewrite the prompt whenever the context
+ * moved. It also tells an agent the shape of a slice it was not given.
+ */
+export declare function memoryInstructions(binding: ResolvedMemoryBinding): string;
+/**
+ * The id on each line is what makes a preference revisable rather than merely
+ * readable: without it the model can read the instruction but cannot name the
+ * thing it would supersede.
+ */
+export declare function renderPreferences(nodes: readonly MemoryNode[]): string;
+//# sourceMappingURL=instructions.d.ts.map

package/dist/memory/instructions.js

@@ -0,0 +1,103 @@
+import { MEMORY_COMMIT_TOOL, MEMORY_FORGET_TOOL, MEMORY_LOAD_TOOL, MEMORY_SEARCH_TOOL, } from "../types.js";
+import { RECOLLECTION_TAG } from "./render.js";
+import { canForget, canWrite } from "./types.js";
+// ---------------------------------------------------------------------------
+// Teaching the model to read and write memory
+//
+// The tool schemas say what the arguments are; none of them says when a thing
+// is worth remembering, or that a recollection is a subgraph rather than a
+// ranked list. That is policy, it is the same for every agent, and it belongs
+// in the system prompt — which is also the only place it can be stated once
+// per run instead of once per call.
+//
+// Both halves go in through `composePrompt`'s `derived` array, so they sit in
+// the cacheable prefix and are recorded in the `system_prompt` node like every
+// other part of what the model read.
+//
+// The text is gated on the binding, not on the project: an agent that cannot
+// write is never told how to commit. A rule it cannot act on is a rule it can
+// only be confused by, and an unusable tool name in the prompt invites a call
+// that will be refused.
+// ---------------------------------------------------------------------------
+export const PREFERENCES_TAG = 'memory-preferences';
+/**
+ * Deliberately not a function of `sees`: those labels come from the run's
+ * context, and naming them here would rewrite the prompt whenever the context
+ * moved. It also tells an agent the shape of a slice it was not given.
+ */
+export function memoryInstructions(binding) {
+    const parts = [READING];
+    if (canWrite(binding)) {
+        parts.push(WRITING);
+    }
+    if (canForget(binding)) {
+        parts.push(FORGETTING);
+    }
+    return parts.join('\n\n');
+}
+const READING = `Memory is a graph shared with other agents, and it outlives this run.
+
+A <${RECOLLECTION_TAG}> block is a subgraph, not a ranked list — the links are
+as much of the answer as the nodes. It is an outline, and the indentation is
+the graph. A line at the left margin reads \`score kind id\` and is something
+the search matched. A line indented under it reads \`<arrow>relation kind id\`
+and is a neighbour: \`\u2192produced\` means the line above produced this one,
+\`\u2190informed\` means this one informed the line above. A neighbour the
+search matched in its own right carries a score too, after the arrow. Under
+each header, indented further, is that node's text — and for a \`file\`, the
+path under /memory you can open or run, its size and how often it has been read.
+
+A line beginning \`+\` is a link to an id already shown above, for an edge the
+outline could not nest. Text is clipped: copy an id into \`${MEMORY_LOAD_TOOL}\`
+to read a node whole. \`${MEMORY_SEARCH_TOOL}\` finds ids, it does not return
+contents.
+
+You see part of the graph, not all of it. Finding nothing means nothing you can
+see, not that it never happened.`;
+const WRITING = `Commit when a run produced something a later run would otherwise have to work
+out again: a script that ran, an interface that behaved differently from its
+documentation, a call sequence that turned out to be necessary. Commit it in
+one \`${MEMORY_COMMIT_TOOL}\` call — the artifact and what explains it are one
+memory, and splitting them leaves a later run holding a script with no idea why
+it exists.
+
+Do not commit the request restated, progress notes, anything you have not
+verified, or anything a search already returned. A node that says what one
+already in memory says is folded into it and its existing id comes back under
+your ref, so the links you asked for still land — on the memory that was
+already there.
+
+Never edit a memory to correct it. Add the new one and link it to the old with
+SUPERSEDES; the wrong answer is evidence too, and the link is what hides it
+from later recalls without destroying the history.
+
+Commit a preference only when the user's own words generalise — "always", "from
+now on", "I prefer", "never". A single request being fulfilled is not a
+standing instruction.`;
+const FORGETTING = `Forget only what is wrong and has no successor. A superseded memory is already
+hidden from recall and should stay for the history, so \`${MEMORY_FORGET_TOOL}\`
+is for what should never have been written, not for what has been replaced.`;
+/**
+ * The id on each line is what makes a preference revisable rather than merely
+ * readable: without it the model can read the instruction but cannot name the
+ * thing it would supersede.
+ */
+export function renderPreferences(nodes) {
+    if (!nodes.length) {
+        return '';
+    }
+    const lines = nodes.map((n) => `- ${oneLine(n.text)} [${n.id}]`);
+    return [
+        `<${PREFERENCES_TAG}>`,
+        "Standing instructions from earlier runs. They apply unless this run's",
+        'request contradicts them; to change one, commit the replacement and link',
+        'it to the id here with SUPERSEDES.',
+        ...lines,
+        `</${PREFERENCES_TAG}>`,
+    ].join('\n');
+}
+/** Never clipped: half an instruction is a different instruction. */
+function oneLine(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+//# sourceMappingURL=instructions.js.map

package/dist/memory/page.d.ts

@@ -0,0 +1,8 @@
+import type { MemoryReport } from './report.ts';
+/** Pinned once, in the run inspector, and shared so the two pages cannot drift. */
+export interface PageOptions {
+    mermaidUrl?: string;
+}
+/** Report → one standalone HTML document. */
+export declare function renderMemoryHtml(report: MemoryReport, opts?: PageOptions): string;
+//# sourceMappingURL=page.d.ts.map