@zenera/neo 1.1.21 → 1.1.23

package/dist/memory/merge.js

@@ -0,0 +1,379 @@
+import { createHash } from 'node:crypto';
+import { readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { hostPath } from "./files.js";
+import { DUPLICATE_SCORE, flatten } from "./index.js";
+import { MANIFEST_FILE, MemoryStore, lockHolder } from "./store.js";
+import { MemoryError } from "./types.js";
+// ---------------------------------------------------------------------------
+// Folding several memories into one
+//
+// Warming a memory is embarrassingly parallel — N runs, N questions, N answers
+// — but the lock is per directory, so N runs cannot share one. They each write
+// their own, and this puts the results back together.
+//
+// Ids are ULIDs and are preserved, which makes the interesting case the one
+// where the same id appears twice: the runs started from a copy of the same
+// memory, so most of what they hold is a shared ancestor rather than a
+// collision. That case reconciles to nothing more than counters, and the
+// counters take the MAX rather than the sum, because merging the same source
+// twice has to be a no-op. Anything that is not a shared ancestor is a real
+// divergence and refuses the whole merge; `force` is the escape hatch, not the
+// default, because picking a winner silently loses a memory.
+//
+// Nothing here embeds. A merge is offline and fast, and the price of that is
+// that every side must already agree on the embedder.
+// ---------------------------------------------------------------------------
+/** How much of a diverged memory's text a conflict report carries. */
+const CLIP = 80;
+/**
+ * Refusal carrying what to look at. Thrown before anything is written, so a
+ * conflicted merge leaves the target exactly as it was.
+ */
+export class MergeConflicts extends MemoryError {
+    conflicts;
+    constructor(conflicts) {
+        super(`${conflicts.length} ${conflicts.length === 1 ? 'memory' : 'memories'} changed in more than one place`, 'compare them with `zen memory show <id>`, or take the newest with --force');
+        this.name = 'MergeConflicts';
+        this.conflicts = conflicts;
+    }
+}
+/**
+ * Fold `sources` into `target`, left to right.
+ *
+ * Order is visible: a memory folded onto a twin keeps the id of whichever copy
+ * arrived first, so `merge a b` and `merge b a` can differ in which id
+ * survives. They never differ in what is remembered.
+ *
+ * On `dryRun` the target is left with the merge applied in memory and nothing
+ * on disk. Report the result and discard the store; do not commit it.
+ */
+export async function mergeMemories(target, sources, opts = {}) {
+    const dedupe = opts.dedupe !== false;
+    const opened = await openSources(target, sources, opts.force === true);
+    const reports = new Map(opened.map((src) => [
+        src,
+        {
+            dir: src.dir,
+            nodes: src.graph.order,
+            edges: src.graph.size,
+            added: 0,
+            shared: 0,
+            twins: 0,
+            files: 0,
+        },
+    ]));
+    const embedding = agree(target, opened);
+    if (embedding) {
+        target.adopt(embedding);
+    }
+    // --- plan: everything refusable is refused here, before a single write ---
+    const plan = new Map();
+    for (const node of target.graph.nodes()) {
+        plan.set(node.id, { node, fresh: false });
+    }
+    const conflicts = [];
+    for (const src of opened) {
+        const report = reports.get(src);
+        for (const node of src.graph.nodes()) {
+            const held = plan.get(node.id);
+            if (!held) {
+                plan.set(node.id, { node, from: src, fresh: true });
+                continue;
+            }
+            if (held.node.revision === node.revision && sameContent(held.node, node)) {
+                plan.set(node.id, { ...held, node: reconcile(held.node, node) });
+                report.shared++;
+                continue;
+            }
+            if (!opts.force) {
+                conflicts.push({
+                    id: node.id,
+                    dir: src.dir,
+                    mine: held.node.revision,
+                    theirs: node.revision,
+                    text: clip(node.text),
+                });
+                continue;
+            }
+            report.shared++;
+            plan.set(node.id, newer(node, held.node)
+                ? { node: reconcile(node, held.node), from: src, fresh: held.fresh }
+                : { ...held, node: reconcile(held.node, node) });
+        }
+    }
+    if (conflicts.length) {
+        throw new MergeConflicts(conflicts);
+    }
+    await checkFiles(plan);
+    // --- fold, in memory ---
+    const before = target.graph.size;
+    const moved = new Map();
+    for (const planned of plan.values()) {
+        if (planned.fresh) {
+            continue;
+        }
+        target.graph.adopt(planned.node);
+        if (planned.from) {
+            carryVector(planned.from, target, planned.node.id);
+        }
+    }
+    const lexical = new Map();
+    for (const node of target.graph.nodes()) {
+        lexical.set(twinKey(node), node.id);
+    }
+    for (const src of opened) {
+        const report = reports.get(src);
+        const stale = target.graph.superseded();
+        for (const node of src.graph.nodes()) {
+            const planned = plan.get(node.id);
+            if (!planned.fresh || planned.from !== src || moved.has(node.id)) {
+                continue;
+            }
+            const twin = dedupe ? twinOf(target, src, planned.node, lexical, stale) : undefined;
+            if (twin) {
+                moved.set(node.id, twin);
+                report.twins++;
+                continue;
+            }
+            target.graph.adopt(planned.node);
+            carryVector(src, target, node.id);
+            lexical.set(twinKey(planned.node), node.id);
+            moved.set(node.id, node.id);
+            report.added++;
+        }
+        for (const link of src.graph.links()) {
+            const from = moved.get(link.source) ?? link.source;
+            const to = moved.get(link.target) ?? link.target;
+            // A self-edge is what a fold leaves behind when both ends collapsed
+            // onto the same memory; it says nothing and is dropped.
+            if (from === to || !target.graph.has(from) || !target.graph.has(to)) {
+                continue;
+            }
+            target.graph.adoptEdge(from, to, link.attrs);
+        }
+    }
+    const report = {
+        dir: target.dir,
+        sources: opened.map((src) => reports.get(src)),
+        added: 0,
+        shared: 0,
+        twins: 0,
+        edges: target.graph.size - before,
+        files: 0,
+        ...(target.embedding ? { embedding: target.embedding } : {}),
+        dryRun: opts.dryRun === true,
+    };
+    if (opts.dryRun) {
+        for (const [id, planned] of plan) {
+            if (copies(id, planned, moved)) {
+                reports.get(planned.from).files++;
+            }
+        }
+        return total(report);
+    }
+    await copyFiles(target, plan, moved, reports);
+    await target.commit();
+    return total(report);
+}
+async function openSources(target, sources, force) {
+    if (!sources.length) {
+        throw new MemoryError('no memories to merge', 'name at least one memory directory');
+    }
+    const here = resolve(target.dir);
+    const seen = new Set();
+    const out = [];
+    for (const dir of sources) {
+        const path = resolve(dir);
+        if (path === here) {
+            throw new MemoryError(`${dir} is the memory being merged into`, 'name only the memories to fold in');
+        }
+        if (seen.has(path)) {
+            throw new MemoryError(`${dir} was named twice`, 'name each memory once');
+        }
+        seen.add(path);
+        // `MemoryStore.open` creates what is missing, so a mistyped path would
+        // otherwise merge an empty memory it had just brought into existence.
+        try {
+            await stat(join(path, MANIFEST_FILE));
+        }
+        catch {
+            throw new MemoryError(`${dir} is not a memory`, `no ${MANIFEST_FILE} in it`);
+        }
+        const held = lockHolder(path);
+        if (held && !force) {
+            throw new MemoryError(`${dir} is in use (pid ${held.pid}, since ${held.startedAt})`, 'wait for that run to finish, or merge with --force');
+        }
+        out.push(await MemoryStore.open(path, { lock: false }));
+    }
+    return out;
+}
+/**
+ * The one embedder everything must already share. Sources that hold nothing are
+ * not asked: a warmup run that found nothing worth keeping never adopted one.
+ */
+function agree(target, sources) {
+    const holding = sources.filter((src) => src.graph.order > 0);
+    const chosen = target.embedding ?? holding.find((src) => src.embedding)?.embedding;
+    for (const src of holding) {
+        const theirs = src.embedding;
+        if (chosen && theirs) {
+            if (theirs.model !== chosen.model || theirs.dimensions !== chosen.dimensions) {
+                throw new MemoryError(`${src.dir} was embedded with ${theirs.model} (${theirs.dimensions}d), ` +
+                    `but this merge is ${chosen.model} (${chosen.dimensions}d)`, 'merge memories warmed with the same embedder');
+            }
+        }
+        else if (chosen) {
+            throw new MemoryError(`${src.dir} has no embedder, but this merge is ${chosen.model} (${chosen.dimensions}d)`, 'its memories would be invisible to search; re-warm it with that embedder');
+        }
+    }
+    return chosen;
+}
+/**
+ * The existing twin rule, applied across memories: same kind, same audience, no
+ * file, not superseded, and either near-identical by cosine or identical once
+ * whitespace and case stop counting.
+ */
+function twinOf(target, src, node, lexical, stale) {
+    if (node.file) {
+        return undefined;
+    }
+    const vector = src.vectors?.get(node.id);
+    const block = target.vectors;
+    if (vector && block && block.dims === vector.length && block.rows) {
+        const allow = (id) => {
+            const other = target.graph.get(id);
+            return !stale.has(id) && !!other && comparable(other, node);
+        };
+        const hit = block.topK(vector, 1, allow)[0];
+        return hit && hit.score >= DUPLICATE_SCORE ? hit.id : undefined;
+    }
+    const id = lexical.get(twinKey(node));
+    return id && !stale.has(id) ? id : undefined;
+}
+function comparable(a, b) {
+    return a.id !== b.id && a.kind === b.kind && !a.file && audienceKey(a) === audienceKey(b);
+}
+function twinKey(node) {
+    return [node.kind, audienceKey(node), flatten(node.text)].join('\u0000');
+}
+function audienceKey(node) {
+    return [...node.audience].sort().join('|');
+}
+function carryVector(from, to, id) {
+    const vector = from.vectors?.get(id);
+    const block = to.vectors;
+    if (vector && block && block.dims === vector.length) {
+        block.set(id, vector);
+    }
+}
+/**
+ * Use is not change: `touch` moves these and deliberately leaves `revision`
+ * alone, which is what lets two runs of the same memory reconcile instead of
+ * colliding. MAX and not sum, so merging a source twice changes nothing.
+ */
+function reconcile(keep, other) {
+    return {
+        ...keep,
+        // ISO-8601 in UTC is fixed width, so string order is time order.
+        createdAt: keep.createdAt <= other.createdAt ? keep.createdAt : other.createdAt,
+        lastUsedAt: keep.lastUsedAt >= other.lastUsedAt ? keep.lastUsedAt : other.lastUsedAt,
+        useCount: Math.max(keep.useCount, other.useCount),
+    };
+}
+function newer(a, b) {
+    return a.revision === b.revision ? a.updatedAt > b.updatedAt : a.revision > b.revision;
+}
+function sameContent(a, b) {
+    return (a.kind === b.kind &&
+        a.text === b.text &&
+        audienceKey(a) === audienceKey(b) &&
+        a.file?.sha256 === b.file?.sha256 &&
+        deep(a.metadata, b.metadata));
+}
+function deep(a, b) {
+    if (a === b) {
+        return true;
+    }
+    if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) {
+        return false;
+    }
+    if (Array.isArray(a) !== Array.isArray(b)) {
+        return false;
+    }
+    const left = a;
+    const right = b;
+    const keys = Object.keys(left);
+    return keys.length === Object.keys(right).length && keys.every((k) => deep(left[k], right[k]));
+}
+/** Stat every file the merge would copy before it copies any of them. */
+async function checkFiles(plan) {
+    for (const planned of plan.values()) {
+        const file = planned.node.file;
+        if (!planned.from || !file) {
+            continue;
+        }
+        const source = hostPath(planned.from.dir, file);
+        const info = await stat(source).catch(() => undefined);
+        if (!info?.isFile()) {
+            throw new MemoryError(`${file.path} is missing from ${planned.from.dir}`, 'that memory is incomplete; re-run it, or forget the node that names the file');
+        }
+    }
+}
+/**
+ * Bytes last, and all or nothing: the digest is checked against the one the
+ * node records, and anything written before a failure is taken back, so a
+ * refused merge cannot leave files no graph mentions.
+ */
+async function copyFiles(target, plan, moved, reports) {
+    const written = [];
+    try {
+        for (const [id, planned] of plan) {
+            const file = planned.node.file;
+            if (!copies(id, planned, moved)) {
+                continue;
+            }
+            const source = hostPath(planned.from.dir, file);
+            const destination = hostPath(target.dir, file);
+            if (source === destination) {
+                continue;
+            }
+            const bytes = await readFile(source);
+            const sha = createHash('sha256').update(bytes).digest('hex');
+            if (sha !== file.sha256) {
+                throw new MemoryError(`${file.path} in ${planned.from.dir} does not match the digest its memory records`, 'that memory was modified outside zen; merge it after re-running it');
+            }
+            const existed = await stat(destination).then(() => true, () => false);
+            await writeFile(destination, bytes);
+            if (!existed) {
+                written.push(destination);
+            }
+            reports.get(planned.from).files++;
+        }
+    }
+    catch (err) {
+        await Promise.all(written.map((path) => rm(path, { force: true })));
+        throw err;
+    }
+}
+/** A file comes across when its node came from a source and survived the fold. */
+function copies(id, planned, moved) {
+    if (!planned.from || !planned.node.file) {
+        return false;
+    }
+    return !planned.fresh || moved.get(id) === id;
+}
+function total(report) {
+    for (const src of report.sources) {
+        report.added += src.added;
+        report.shared += src.shared;
+        report.twins += src.twins;
+        report.files += src.files;
+    }
+    return report;
+}
+function clip(text) {
+    const flat = flatten(text);
+    return flat.length > CLIP ? `${flat.slice(0, CLIP - 1)}…` : flat;
+}
+//# sourceMappingURL=merge.js.map

package/dist/memory/store.d.ts

@@ -19,6 +19,11 @@ export interface MemoryManifest {
     edges: number;
     updatedAt: string;
 }
+export interface Lock {
+    pid: number;
+    host: string;
+    startedAt: string;
+}
 export interface OpenOptions {
     /** the embedder the caller intends to use; a mismatch with the manifest is fatal */
     embedding?: MemoryEmbedding;
@@ -46,4 +51,10 @@ export declare class MemoryStore {
     commit(): Promise<void>;
     release(): void;
 }
+/**
+ * Who is writing a memory right now, if anyone. Read-only on purpose: it never
+ * claims the lock and never clears a stale one, so it is safe to ask about a
+ * directory this process has no intention of opening for writing.
+ */
+export declare function lockHolder(dir: string): Lock | undefined;
 //# sourceMappingURL=store.d.ts.map

package/dist/memory/store.js

@@ -202,6 +202,15 @@ async function atomic(path, body) {
         throw err;
     }
 }
+/**
+ * Who is writing a memory right now, if anyone. Read-only on purpose: it never
+ * claims the lock and never clears a stale one, so it is safe to ask about a
+ * directory this process has no intention of opening for writing.
+ */
+export function lockHolder(dir) {
+    const held = readLock(join(dir, LOCK_FILE));
+    return held && held.host === hostname() && alive(held.pid) ? held : undefined;
+}
 function readLock(path) {
     try {
         return JSON.parse(readFileSync(path, 'utf8'));

package/dist/memory/tools.js

@@ -161,11 +161,11 @@ export function memoryTools(opts) {
                         properties: {
                             ref: {
                                 type: 'string',
-                                description: 'a short name for a new node, used by edges in this call',
+                                description: 'a short local name for a new node (e.g. "task_1", "file_1"), used by edges in this call. Use ref (not id) when creating new nodes.',
                             },
                             id: {
                                 type: 'string',
-                                description: 'to change an existing node instead of adding one',
+                                description: 'the permanent id of an existing node to update in place (from search/load). Do not use for new nodes; use ref instead.',
                             },
                             kind: { type: 'string', enum: kinds, description: kindHelp },
                             text: {
@@ -173,8 +173,25 @@ export function memoryTools(opts) {
                                 description: 'what to remember, in your own words',
                             },
                             file: {
-                                type: 'string',
-                                description: 'path of a file to keep a copy of, for kind "file"',
+                                oneOf: [
+                                    {
+                                        type: 'string',
+                                        description: 'path of an existing file in the workspace to keep a copy of (e.g. "report.py" or "/workspace/report.py"), for kind "file"',
+                                    },
+                                    {
+                                        type: 'object',
+                                        properties: {
+                                            path: {
+                                                type: 'string',
+                                                description: 'path of an existing file in the workspace',
+                                            },
+                                        },
+                                        required: ['path'],
+                                        additionalProperties: false,
+                                        description: 'file object with path',
+                                    },
+                                ],
+                                description: 'path of an existing file in the workspace to keep a copy of (e.g. "report.py" or "/workspace/report.py"), for kind "file"',
                             },
                             ...audience,
                             expected_revision: {
@@ -192,11 +209,11 @@ export function memoryTools(opts) {
                         properties: {
                             from: {
                                 type: 'string',
-                                description: 'a ref from this call, or an id',
+                                description: 'a ref from this call, or an existing node id',
                             },
                             to: {
                                 type: 'string',
-                                description: 'a ref from this call, or an id',
+                                description: 'a ref from this call, or an existing node id',
                             },
                             relation: {
                                 type: 'string',
@@ -216,15 +233,28 @@ export function memoryTools(opts) {
             const opId = memoryOpId(tc.state.runId, tc.callId);
             let nodes;
             try {
-                nodes = args.nodes.map((n) => ({
-                    ref: n.ref,
-                    id: n.id,
-                    kind: n.kind,
-                    text: n.text,
-                    audience: n.audience,
-                    expectedRevision: n.expected_revision,
-                    file: n.file === undefined ? undefined : { source: source(tc, n.file) },
-                }));
+                nodes = args.nodes.map((n) => {
+                    let fileSpec;
+                    if (n.file !== undefined) {
+                        const rawPath = extractFilePath(n.file);
+                        if (!rawPath.trim()) {
+                            throw new MemoryError('file path is required', 'pass the path of the file in the workspace to remember, e.g. "report.py"');
+                        }
+                        fileSpec = {
+                            source: source(tc, rawPath),
+                            path: rawPath,
+                        };
+                    }
+                    return {
+                        ref: n.ref,
+                        id: n.id,
+                        kind: n.kind,
+                        text: n.text,
+                        audience: n.audience,
+                        expectedRevision: n.expected_revision,
+                        file: fileSpec,
+                    };
+                });
             }
             catch (err) {
                 return refusal(err);
@@ -322,11 +352,31 @@ function refusal(err) {
     }
     throw err;
 }
+function extractFilePath(raw) {
+    if (typeof raw === 'string') {
+        return raw;
+    }
+    if (raw &&
+        typeof raw === 'object' &&
+        'path' in raw &&
+        typeof raw.path === 'string') {
+        return raw.path;
+    }
+    return '';
+}
 function source(tc, path) {
     if (!tc.services.resolveFile) {
         throw new MemoryError('this agent cannot remember files', 'commit the node without `file`, or give the agent a workspace');
     }
-    return tc.services.resolveFile(path);
+    try {
+        return tc.services.resolveFile(path);
+    }
+    catch (err) {
+        if (err instanceof MemoryError) {
+            throw err;
+        }
+        throw new MemoryError(err instanceof Error ? err.message : String(err), 'remember a file the run actually wrote, by the path the file tools use');
+    }
 }
 function changed(index, ids) {
     return Object.values(ids).flatMap((id) => {

package/dist/memory/vectors.d.ts

@@ -9,6 +9,8 @@ export declare class VectorBlock {
     get rows(): number;
     get ids(): readonly string[];
     has(id: string): boolean;
+    /** A copy, not a view: the row it came from moves the next time something is deleted. */
+    get(id: string): Float32Array | undefined;
     set(id: string, vector: ArrayLike<number>): void;
     remove(id: string): boolean;
     /** Exact kNN over the rows `allow` accepts. Ties break toward the earlier row. */

package/dist/memory/vectors.js

@@ -39,6 +39,13 @@ export class VectorBlock {
     has(id) {
         return this.#index.has(id);
     }
+    /** A copy, not a view: the row it came from moves the next time something is deleted. */
+    get(id) {
+        const row = this.#index.get(id);
+        return row === undefined
+            ? undefined
+            : this.#data.slice(row * this.#dims, (row + 1) * this.#dims);
+    }
     set(id, vector) {
         if (vector.length !== this.#dims) {
             throw new Error(`expected a ${this.#dims}-dimension vector, got ${vector.length}`);

package/dist/project/config.d.ts

@@ -169,6 +169,10 @@ declare const sandbox: z.ZodObject<{
     }>>;
     workdir: z.ZodOptional<z.ZodString>;
     timeout: z.ZodOptional<z.ZodInt>;
+    hardening: z.ZodOptional<z.ZodEnum<{
+        strict: "strict";
+        standard: "standard";
+    }>>;
     user: z.ZodOptional<z.ZodString>;
     persist: z.ZodOptional<z.ZodBoolean>;
     env: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -295,6 +299,10 @@ export declare const projectSchema: z.ZodObject<{
         }>>;
         workdir: z.ZodOptional<z.ZodString>;
         timeout: z.ZodOptional<z.ZodInt>;
+        hardening: z.ZodOptional<z.ZodEnum<{
+            strict: "strict";
+            standard: "standard";
+        }>>;
         user: z.ZodOptional<z.ZodString>;
         persist: z.ZodOptional<z.ZodBoolean>;
         env: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -355,6 +363,10 @@ export declare const projectSchema: z.ZodObject<{
             }>>;
             workdir: z.ZodOptional<z.ZodString>;
             timeout: z.ZodOptional<z.ZodInt>;
+            hardening: z.ZodOptional<z.ZodEnum<{
+                strict: "strict";
+                standard: "standard";
+            }>>;
             user: z.ZodOptional<z.ZodString>;
             persist: z.ZodOptional<z.ZodBoolean>;
             env: z.ZodOptional<z.ZodArray<z.ZodString>>;

package/dist/project/config.js

@@ -263,6 +263,20 @@ const sandbox = z
     workdir: z.string().min(1).optional(),
     /** seconds one command may take before it is killed */
     timeout: z.int().positive().optional(),
+    /**
+     * How much of the host the container is allowed to be.
+     *
+     * `standard` (the default) keeps the kernel capabilities and a
+     * writable root filesystem, so `apt` and `pip` work — which is what a
+     * project whose agents install their own dependencies needs.
+     *
+     * `strict` drops every capability, mounts the root filesystem
+     * read-only with an in-memory `noexec` `/tmp`, runs as an
+     * unprivileged uid and turns the network off unless this block asks
+     * for one. Nothing can be installed at run time, so the image has to
+     * carry everything the agent will need. It requires rootless podman.
+     */
+    hardening: z.enum(['standard', 'strict']).optional(),
     /** uid, name or `uid:gid`; unset means the image's own user */
     user: z.string().min(1).optional(),
     /** keep the container between sessions instead of removing it */