@zenera/neo 1.1.22 → 1.1.23

package/dist/index.d.ts

@@ -12,6 +12,7 @@ export * as Kernel from './kernel.ts';
 export * from './memory/files.ts';
 export * from './memory/graph.ts';
 export * from './memory/index.ts';
+export * from './memory/merge.ts';
 export * from './memory/page.ts';
 export * from './memory/recall.ts';
 export * from './memory/render.ts';

package/dist/index.js

@@ -12,6 +12,7 @@ export * as Kernel from "./kernel.js";
 export * from "./memory/files.js";
 export * from "./memory/graph.js";
 export * from "./memory/index.js";
+export * from "./memory/merge.js";
 export * from "./memory/page.js";
 export * from "./memory/recall.js";
 export * from "./memory/render.js";

package/dist/memory/graph.d.ts

@@ -29,14 +29,28 @@ export declare class MemoryGraph {
     has(id: string): boolean;
     get(id: string): MemoryNode | undefined;
     add(draft: NodeDraft, at: string): MemoryNode;
+    /**
+     * Put a node in exactly as it stands — id, timestamps, revision and counters
+     * included. `add` mints a new memory; this carries one across from another
+     * graph, and replaces the node already under that id if there is one.
+     */
+    adopt(node: MemoryNode): void;
     update(id: string, patch: NodePatch, at: string): MemoryNode;
     /** Reading is what counts as use — see `memory_load`, not `memory_search`. */
     touch(id: string, at: string): MemoryNode | undefined;
     link(source: string, target: string, relation: Relation, at: string): void;
+    /** `link` for an edge that already exists elsewhere: it keeps the stamp it had. */
+    adoptEdge(source: string, target: string, attrs: MemoryEdgeAttrs): void;
     /** Drops the node and every edge touching it; the file it names is the store's to unlink. */
     forget(id: string): MemoryNode | undefined;
     nodes(sees?: readonly string[]): MemoryNode[];
     edges(sees?: readonly string[]): MemoryEdge[];
+    /** Edges with their attributes and no mask — `edges` is the view, this is the record. */
+    links(): {
+        source: string;
+        target: string;
+        attrs: MemoryEdgeAttrs;
+    }[];
     /** Both directions: a memory is as often reached from its artifact as from its request. */
     neighbors(id: string, sees?: readonly string[]): MemoryEdge[];
     /** Edges between nodes already chosen — what makes a materialized subgraph closed. */

package/dist/memory/graph.js

@@ -41,6 +41,20 @@ export class MemoryGraph {
         this.#graph.addNode(node.id, node);
         return node;
     }
+    /**
+     * Put a node in exactly as it stands — id, timestamps, revision and counters
+     * included. `add` mints a new memory; this carries one across from another
+     * graph, and replaces the node already under that id if there is one.
+     */
+    adopt(node) {
+        const next = { ...node };
+        if (this.#graph.hasNode(node.id)) {
+            this.#graph.replaceNodeAttributes(node.id, next);
+        }
+        else {
+            this.#graph.addNode(node.id, next);
+        }
+    }
     update(id, patch, at) {
         const current = this.#graph.getNodeAttributes(id);
         if (patch.expectedRevision !== undefined && patch.expectedRevision !== current.revision) {
@@ -75,6 +89,12 @@ export class MemoryGraph {
             createdAt: at,
         });
     }
+    /** `link` for an edge that already exists elsewhere: it keeps the stamp it had. */
+    adoptEdge(source, target, attrs) {
+        this.#graph.mergeEdgeWithKey(edgeKey(source, target, attrs.relation), source, target, {
+            ...attrs,
+        });
+    }
     /** Drops the node and every edge touching it; the file it names is the store's to unlink. */
     forget(id) {
         const node = this.get(id);
@@ -102,6 +122,14 @@ export class MemoryGraph {
         });
         return out;
     }
+    /** Edges with their attributes and no mask — `edges` is the view, this is the record. */
+    links() {
+        const out = [];
+        this.#graph.forEachEdge((_key, attrs, source, target) => {
+            out.push({ source, target, attrs });
+        });
+        return out;
+    }
     /** Both directions: a memory is as often reached from its artifact as from its request. */
     neighbors(id, sees) {
         if (!this.#graph.hasNode(id)) {

package/dist/memory/index.d.ts

@@ -91,4 +91,6 @@ export declare class MemoryIndex {
     commit(tx: MemoryTransaction, opts: CommitOptions): Promise<CommitResult>;
     forget(ids: readonly string[], sees: readonly string[]): Promise<MemoryNode[]>;
 }
+/** Whitespace and case are not what makes two memories different. */
+export declare function flatten(text: string): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/memory/index.js

@@ -383,7 +383,7 @@ export class MemoryIndex {
     }
 }
 /** Whitespace and case are not what makes two memories different. */
-function flatten(text) {
+export function flatten(text) {
     return text.replace(/\s+/g, ' ').trim().toLowerCase();
 }
 //# sourceMappingURL=index.js.map

package/dist/memory/merge.d.ts

@@ -0,0 +1,65 @@
+import { MemoryStore, type MemoryEmbedding } from './store.ts';
+import { MemoryError } from './types.ts';
+export interface MergeOptions {
+    /** fold near-identical memories onto the one already there; on by default */
+    dedupe?: boolean;
+    /** resolve divergence by taking the highest revision instead of refusing */
+    force?: boolean;
+    /** plan and report, write nothing */
+    dryRun?: boolean;
+}
+export interface MergeSource {
+    dir: string;
+    /** what the source holds */
+    nodes: number;
+    edges: number;
+    /** what it contributed */
+    added: number;
+    /** ids the target already had, reconciled rather than duplicated */
+    shared: number;
+    /** new ids that turned out to be a memory the target already held */
+    twins: number;
+    files: number;
+}
+export interface MergeConflict {
+    id: string;
+    /** the source that disagreed */
+    dir: string;
+    /** revision already in the merge, and the one the source brought */
+    mine: number;
+    theirs: number;
+    text: string;
+}
+export interface MergeReport {
+    /** the memory everything was folded into */
+    dir: string;
+    sources: MergeSource[];
+    added: number;
+    shared: number;
+    twins: number;
+    /** edges the merge added, after re-pointing folded ends */
+    edges: number;
+    files: number;
+    embedding?: MemoryEmbedding;
+    dryRun: boolean;
+}
+/**
+ * Refusal carrying what to look at. Thrown before anything is written, so a
+ * conflicted merge leaves the target exactly as it was.
+ */
+export declare class MergeConflicts extends MemoryError {
+    readonly conflicts: readonly MergeConflict[];
+    constructor(conflicts: readonly MergeConflict[]);
+}
+/**
+ * Fold `sources` into `target`, left to right.
+ *
+ * Order is visible: a memory folded onto a twin keeps the id of whichever copy
+ * arrived first, so `merge a b` and `merge b a` can differ in which id
+ * survives. They never differ in what is remembered.
+ *
+ * On `dryRun` the target is left with the merge applied in memory and nothing
+ * on disk. Report the result and discard the store; do not commit it.
+ */
+export declare function mergeMemories(target: MemoryStore, sources: readonly string[], opts?: MergeOptions): Promise<MergeReport>;
+//# sourceMappingURL=merge.d.ts.map

package/dist/memory/merge.js

@@ -0,0 +1,379 @@
+import { createHash } from 'node:crypto';
+import { readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { hostPath } from "./files.js";
+import { DUPLICATE_SCORE, flatten } from "./index.js";
+import { MANIFEST_FILE, MemoryStore, lockHolder } from "./store.js";
+import { MemoryError } from "./types.js";
+// ---------------------------------------------------------------------------
+// Folding several memories into one
+//
+// Warming a memory is embarrassingly parallel — N runs, N questions, N answers
+// — but the lock is per directory, so N runs cannot share one. They each write
+// their own, and this puts the results back together.
+//
+// Ids are ULIDs and are preserved, which makes the interesting case the one
+// where the same id appears twice: the runs started from a copy of the same
+// memory, so most of what they hold is a shared ancestor rather than a
+// collision. That case reconciles to nothing more than counters, and the
+// counters take the MAX rather than the sum, because merging the same source
+// twice has to be a no-op. Anything that is not a shared ancestor is a real
+// divergence and refuses the whole merge; `force` is the escape hatch, not the
+// default, because picking a winner silently loses a memory.
+//
+// Nothing here embeds. A merge is offline and fast, and the price of that is
+// that every side must already agree on the embedder.
+// ---------------------------------------------------------------------------
+/** How much of a diverged memory's text a conflict report carries. */
+const CLIP = 80;
+/**
+ * Refusal carrying what to look at. Thrown before anything is written, so a
+ * conflicted merge leaves the target exactly as it was.
+ */
+export class MergeConflicts extends MemoryError {
+    conflicts;
+    constructor(conflicts) {
+        super(`${conflicts.length} ${conflicts.length === 1 ? 'memory' : 'memories'} changed in more than one place`, 'compare them with `zen memory show <id>`, or take the newest with --force');
+        this.name = 'MergeConflicts';
+        this.conflicts = conflicts;
+    }
+}
+/**
+ * Fold `sources` into `target`, left to right.
+ *
+ * Order is visible: a memory folded onto a twin keeps the id of whichever copy
+ * arrived first, so `merge a b` and `merge b a` can differ in which id
+ * survives. They never differ in what is remembered.
+ *
+ * On `dryRun` the target is left with the merge applied in memory and nothing
+ * on disk. Report the result and discard the store; do not commit it.
+ */
+export async function mergeMemories(target, sources, opts = {}) {
+    const dedupe = opts.dedupe !== false;
+    const opened = await openSources(target, sources, opts.force === true);
+    const reports = new Map(opened.map((src) => [
+        src,
+        {
+            dir: src.dir,
+            nodes: src.graph.order,
+            edges: src.graph.size,
+            added: 0,
+            shared: 0,
+            twins: 0,
+            files: 0,
+        },
+    ]));
+    const embedding = agree(target, opened);
+    if (embedding) {
+        target.adopt(embedding);
+    }
+    // --- plan: everything refusable is refused here, before a single write ---
+    const plan = new Map();
+    for (const node of target.graph.nodes()) {
+        plan.set(node.id, { node, fresh: false });
+    }
+    const conflicts = [];
+    for (const src of opened) {
+        const report = reports.get(src);
+        for (const node of src.graph.nodes()) {
+            const held = plan.get(node.id);
+            if (!held) {
+                plan.set(node.id, { node, from: src, fresh: true });
+                continue;
+            }
+            if (held.node.revision === node.revision && sameContent(held.node, node)) {
+                plan.set(node.id, { ...held, node: reconcile(held.node, node) });
+                report.shared++;
+                continue;
+            }
+            if (!opts.force) {
+                conflicts.push({
+                    id: node.id,
+                    dir: src.dir,
+                    mine: held.node.revision,
+                    theirs: node.revision,
+                    text: clip(node.text),
+                });
+                continue;
+            }
+            report.shared++;
+            plan.set(node.id, newer(node, held.node)
+                ? { node: reconcile(node, held.node), from: src, fresh: held.fresh }
+                : { ...held, node: reconcile(held.node, node) });
+        }
+    }
+    if (conflicts.length) {
+        throw new MergeConflicts(conflicts);
+    }
+    await checkFiles(plan);
+    // --- fold, in memory ---
+    const before = target.graph.size;
+    const moved = new Map();
+    for (const planned of plan.values()) {
+        if (planned.fresh) {
+            continue;
+        }
+        target.graph.adopt(planned.node);
+        if (planned.from) {
+            carryVector(planned.from, target, planned.node.id);
+        }
+    }
+    const lexical = new Map();
+    for (const node of target.graph.nodes()) {
+        lexical.set(twinKey(node), node.id);
+    }
+    for (const src of opened) {
+        const report = reports.get(src);
+        const stale = target.graph.superseded();
+        for (const node of src.graph.nodes()) {
+            const planned = plan.get(node.id);
+            if (!planned.fresh || planned.from !== src || moved.has(node.id)) {
+                continue;
+            }
+            const twin = dedupe ? twinOf(target, src, planned.node, lexical, stale) : undefined;
+            if (twin) {
+                moved.set(node.id, twin);
+                report.twins++;
+                continue;
+            }
+            target.graph.adopt(planned.node);
+            carryVector(src, target, node.id);
+            lexical.set(twinKey(planned.node), node.id);
+            moved.set(node.id, node.id);
+            report.added++;
+        }
+        for (const link of src.graph.links()) {
+            const from = moved.get(link.source) ?? link.source;
+            const to = moved.get(link.target) ?? link.target;
+            // A self-edge is what a fold leaves behind when both ends collapsed
+            // onto the same memory; it says nothing and is dropped.
+            if (from === to || !target.graph.has(from) || !target.graph.has(to)) {
+                continue;
+            }
+            target.graph.adoptEdge(from, to, link.attrs);
+        }
+    }
+    const report = {
+        dir: target.dir,
+        sources: opened.map((src) => reports.get(src)),
+        added: 0,
+        shared: 0,
+        twins: 0,
+        edges: target.graph.size - before,
+        files: 0,
+        ...(target.embedding ? { embedding: target.embedding } : {}),
+        dryRun: opts.dryRun === true,
+    };
+    if (opts.dryRun) {
+        for (const [id, planned] of plan) {
+            if (copies(id, planned, moved)) {
+                reports.get(planned.from).files++;
+            }
+        }
+        return total(report);
+    }
+    await copyFiles(target, plan, moved, reports);
+    await target.commit();
+    return total(report);
+}
+async function openSources(target, sources, force) {
+    if (!sources.length) {
+        throw new MemoryError('no memories to merge', 'name at least one memory directory');
+    }
+    const here = resolve(target.dir);
+    const seen = new Set();
+    const out = [];
+    for (const dir of sources) {
+        const path = resolve(dir);
+        if (path === here) {
+            throw new MemoryError(`${dir} is the memory being merged into`, 'name only the memories to fold in');
+        }
+        if (seen.has(path)) {
+            throw new MemoryError(`${dir} was named twice`, 'name each memory once');
+        }
+        seen.add(path);
+        // `MemoryStore.open` creates what is missing, so a mistyped path would
+        // otherwise merge an empty memory it had just brought into existence.
+        try {
+            await stat(join(path, MANIFEST_FILE));
+        }
+        catch {
+            throw new MemoryError(`${dir} is not a memory`, `no ${MANIFEST_FILE} in it`);
+        }
+        const held = lockHolder(path);
+        if (held && !force) {
+            throw new MemoryError(`${dir} is in use (pid ${held.pid}, since ${held.startedAt})`, 'wait for that run to finish, or merge with --force');
+        }
+        out.push(await MemoryStore.open(path, { lock: false }));
+    }
+    return out;
+}
+/**
+ * The one embedder everything must already share. Sources that hold nothing are
+ * not asked: a warmup run that found nothing worth keeping never adopted one.
+ */
+function agree(target, sources) {
+    const holding = sources.filter((src) => src.graph.order > 0);
+    const chosen = target.embedding ?? holding.find((src) => src.embedding)?.embedding;
+    for (const src of holding) {
+        const theirs = src.embedding;
+        if (chosen && theirs) {
+            if (theirs.model !== chosen.model || theirs.dimensions !== chosen.dimensions) {
+                throw new MemoryError(`${src.dir} was embedded with ${theirs.model} (${theirs.dimensions}d), ` +
+                    `but this merge is ${chosen.model} (${chosen.dimensions}d)`, 'merge memories warmed with the same embedder');
+            }
+        }
+        else if (chosen) {
+            throw new MemoryError(`${src.dir} has no embedder, but this merge is ${chosen.model} (${chosen.dimensions}d)`, 'its memories would be invisible to search; re-warm it with that embedder');
+        }
+    }
+    return chosen;
+}
+/**
+ * The existing twin rule, applied across memories: same kind, same audience, no
+ * file, not superseded, and either near-identical by cosine or identical once
+ * whitespace and case stop counting.
+ */
+function twinOf(target, src, node, lexical, stale) {
+    if (node.file) {
+        return undefined;
+    }
+    const vector = src.vectors?.get(node.id);
+    const block = target.vectors;
+    if (vector && block && block.dims === vector.length && block.rows) {
+        const allow = (id) => {
+            const other = target.graph.get(id);
+            return !stale.has(id) && !!other && comparable(other, node);
+        };
+        const hit = block.topK(vector, 1, allow)[0];
+        return hit && hit.score >= DUPLICATE_SCORE ? hit.id : undefined;
+    }
+    const id = lexical.get(twinKey(node));
+    return id && !stale.has(id) ? id : undefined;
+}
+function comparable(a, b) {
+    return a.id !== b.id && a.kind === b.kind && !a.file && audienceKey(a) === audienceKey(b);
+}
+function twinKey(node) {
+    return [node.kind, audienceKey(node), flatten(node.text)].join('\u0000');
+}
+function audienceKey(node) {
+    return [...node.audience].sort().join('|');
+}
+function carryVector(from, to, id) {
+    const vector = from.vectors?.get(id);
+    const block = to.vectors;
+    if (vector && block && block.dims === vector.length) {
+        block.set(id, vector);
+    }
+}
+/**
+ * Use is not change: `touch` moves these and deliberately leaves `revision`
+ * alone, which is what lets two runs of the same memory reconcile instead of
+ * colliding. MAX and not sum, so merging a source twice changes nothing.
+ */
+function reconcile(keep, other) {
+    return {
+        ...keep,
+        // ISO-8601 in UTC is fixed width, so string order is time order.
+        createdAt: keep.createdAt <= other.createdAt ? keep.createdAt : other.createdAt,
+        lastUsedAt: keep.lastUsedAt >= other.lastUsedAt ? keep.lastUsedAt : other.lastUsedAt,
+        useCount: Math.max(keep.useCount, other.useCount),
+    };
+}
+function newer(a, b) {
+    return a.revision === b.revision ? a.updatedAt > b.updatedAt : a.revision > b.revision;
+}
+function sameContent(a, b) {
+    return (a.kind === b.kind &&
+        a.text === b.text &&
+        audienceKey(a) === audienceKey(b) &&
+        a.file?.sha256 === b.file?.sha256 &&
+        deep(a.metadata, b.metadata));
+}
+function deep(a, b) {
+    if (a === b) {
+        return true;
+    }
+    if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) {
+        return false;
+    }
+    if (Array.isArray(a) !== Array.isArray(b)) {
+        return false;
+    }
+    const left = a;
+    const right = b;
+    const keys = Object.keys(left);
+    return keys.length === Object.keys(right).length && keys.every((k) => deep(left[k], right[k]));
+}
+/** Stat every file the merge would copy before it copies any of them. */
+async function checkFiles(plan) {
+    for (const planned of plan.values()) {
+        const file = planned.node.file;
+        if (!planned.from || !file) {
+            continue;
+        }
+        const source = hostPath(planned.from.dir, file);
+        const info = await stat(source).catch(() => undefined);
+        if (!info?.isFile()) {
+            throw new MemoryError(`${file.path} is missing from ${planned.from.dir}`, 'that memory is incomplete; re-run it, or forget the node that names the file');
+        }
+    }
+}
+/**
+ * Bytes last, and all or nothing: the digest is checked against the one the
+ * node records, and anything written before a failure is taken back, so a
+ * refused merge cannot leave files no graph mentions.
+ */
+async function copyFiles(target, plan, moved, reports) {
+    const written = [];
+    try {
+        for (const [id, planned] of plan) {
+            const file = planned.node.file;
+            if (!copies(id, planned, moved)) {
+                continue;
+            }
+            const source = hostPath(planned.from.dir, file);
+            const destination = hostPath(target.dir, file);
+            if (source === destination) {
+                continue;
+            }
+            const bytes = await readFile(source);
+            const sha = createHash('sha256').update(bytes).digest('hex');
+            if (sha !== file.sha256) {
+                throw new MemoryError(`${file.path} in ${planned.from.dir} does not match the digest its memory records`, 'that memory was modified outside zen; merge it after re-running it');
+            }
+            const existed = await stat(destination).then(() => true, () => false);
+            await writeFile(destination, bytes);
+            if (!existed) {
+                written.push(destination);
+            }
+            reports.get(planned.from).files++;
+        }
+    }
+    catch (err) {
+        await Promise.all(written.map((path) => rm(path, { force: true })));
+        throw err;
+    }
+}
+/** A file comes across when its node came from a source and survived the fold. */
+function copies(id, planned, moved) {
+    if (!planned.from || !planned.node.file) {
+        return false;
+    }
+    return !planned.fresh || moved.get(id) === id;
+}
+function total(report) {
+    for (const src of report.sources) {
+        report.added += src.added;
+        report.shared += src.shared;
+        report.twins += src.twins;
+        report.files += src.files;
+    }
+    return report;
+}
+function clip(text) {
+    const flat = flatten(text);
+    return flat.length > CLIP ? `${flat.slice(0, CLIP - 1)}…` : flat;
+}
+//# sourceMappingURL=merge.js.map

package/dist/memory/store.d.ts

@@ -19,6 +19,11 @@ export interface MemoryManifest {
     edges: number;
     updatedAt: string;
 }
+export interface Lock {
+    pid: number;
+    host: string;
+    startedAt: string;
+}
 export interface OpenOptions {
     /** the embedder the caller intends to use; a mismatch with the manifest is fatal */
     embedding?: MemoryEmbedding;
@@ -46,4 +51,10 @@ export declare class MemoryStore {
     commit(): Promise<void>;
     release(): void;
 }
+/**
+ * Who is writing a memory right now, if anyone. Read-only on purpose: it never
+ * claims the lock and never clears a stale one, so it is safe to ask about a
+ * directory this process has no intention of opening for writing.
+ */
+export declare function lockHolder(dir: string): Lock | undefined;
 //# sourceMappingURL=store.d.ts.map

package/dist/memory/store.js

@@ -202,6 +202,15 @@ async function atomic(path, body) {
         throw err;
     }
 }
+/**
+ * Who is writing a memory right now, if anyone. Read-only on purpose: it never
+ * claims the lock and never clears a stale one, so it is safe to ask about a
+ * directory this process has no intention of opening for writing.
+ */
+export function lockHolder(dir) {
+    const held = readLock(join(dir, LOCK_FILE));
+    return held && held.host === hostname() && alive(held.pid) ? held : undefined;
+}
 function readLock(path) {
     try {
         return JSON.parse(readFileSync(path, 'utf8'));

package/dist/memory/vectors.d.ts

@@ -9,6 +9,8 @@ export declare class VectorBlock {
     get rows(): number;
     get ids(): readonly string[];
     has(id: string): boolean;
+    /** A copy, not a view: the row it came from moves the next time something is deleted. */
+    get(id: string): Float32Array | undefined;
     set(id: string, vector: ArrayLike<number>): void;
     remove(id: string): boolean;
     /** Exact kNN over the rows `allow` accepts. Ties break toward the earlier row. */

package/dist/memory/vectors.js

@@ -39,6 +39,13 @@ export class VectorBlock {
     has(id) {
         return this.#index.has(id);
     }
+    /** A copy, not a view: the row it came from moves the next time something is deleted. */
+    get(id) {
+        const row = this.#index.get(id);
+        return row === undefined
+            ? undefined
+            : this.#data.slice(row * this.#dims, (row + 1) * this.#dims);
+    }
     set(id, vector) {
         if (vector.length !== this.#dims) {
             throw new Error(`expected a ${this.#dims}-dimension vector, got ${vector.length}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/neo",
-    "version": "1.1.22",
+    "version": "1.1.23",
     "description": "A multi-agent runtime for Node.js: agents, models, tools, skills, memory and an append-only trajectory.",
     "keywords": [
         "agents",