@zenera/neo 1.1.22 → 1.1.24

package/dist/memory/grep.d.ts

@@ -0,0 +1,57 @@
+import { type MatchOptions } from './match.ts';
+import type { Omitted } from './report.ts';
+import type { MemoryStore } from './store.ts';
+import { type MemoryNode } from './types.ts';
+/** The three places a pattern is looked for. */
+export declare const GREP_FIELDS: readonly ["text", "metadata", "file"];
+export type GrepField = (typeof GREP_FIELDS)[number];
+/** What to do about nodes something has superseded. */
+export type StaleMode = 'exclude' | 'include' | 'only';
+/** Past this a remembered file is reported as unsearched rather than read. */
+export declare const MAX_GREP_FILE_BYTES: number;
+/** And past this, so does the rest of the scan: one memory can hold hundreds. */
+export declare const MAX_GREP_BYTES: number;
+export interface GrepHit {
+    where: GrepField;
+    /** 1-based, within that field */
+    line: number;
+    /** the matched line, clipped around the match */
+    text: string;
+}
+export interface MemoryMatch {
+    node: MemoryNode;
+    hits: GrepHit[];
+    /** hits this node had beyond the ones kept */
+    more: number;
+    /** superseded by a newer node; only ever true when they were asked for */
+    stale: boolean;
+}
+/** A remembered file that was not searched, so absence of a hit proves nothing. */
+export interface SkippedFile {
+    id: string;
+    /** the mount path, as the agent knows it */
+    path: string;
+    reason: Omitted;
+}
+export interface MemoryGrepFilter extends MatchOptions {
+    /** audience labels the reader holds; absent reads the graph unmasked */
+    sees?: readonly string[];
+    kinds?: readonly string[];
+    audience?: string;
+    /** only nodes that remember a file */
+    files?: boolean;
+    /** which fields to read; all three by default */
+    in?: readonly GrepField[];
+    stale?: StaleMode;
+    limit?: number;
+    hitsPerNode?: number;
+}
+export interface MemoryGrepResult {
+    /** how many nodes matched, whatever was kept */
+    found: number;
+    matches: MemoryMatch[];
+    truncated: boolean;
+    skipped: SkippedFile[];
+}
+export declare function grepMemory(store: MemoryStore, pattern: string, filter?: MemoryGrepFilter): Promise<MemoryGrepResult>;
+//# sourceMappingURL=grep.d.ts.map

package/dist/memory/grep.js

@@ -0,0 +1,182 @@
+import { readFile, stat } from 'node:fs/promises';
+import { hostPath } from "./files.js";
+import { PatternError, matcher } from "./match.js";
+import { MemoryError } from "./types.js";
+// ---------------------------------------------------------------------------
+// The memory, read exhaustively
+//
+// `search` ranks and `grep` decides. They answer different questions and
+// neither can be made to answer the other's: a ranking hands back the top of a
+// list, so "nothing came back" and "nothing is there" are the same result, and
+// the one question memory keeps raising — *did we already write this down* —
+// needs them to be different results. So this is one pass, no embedder, no
+// credential, and `found` is the true total even when the list was cut.
+//
+// Three fields, because a memory is not only its summary. `text` is what the
+// ranker sees; `metadata` is where an agent puts the identifiers nobody would
+// ever phrase as a sentence — a commit sha, an endpoint, a container name; and
+// the bytes of a remembered file are the thing itself. An exact match belongs
+// against all three.
+//
+// Two rules keep this honest against raw `grep` over /memory, which is what it
+// exists to replace. Superseded nodes are excluded unless asked for, and come
+// back flagged when they are — serving a withdrawn correction as current is
+// the failure mode the mount is warned about. And reading here is not use:
+// recency decay is meant to track what an agent actually opened, and a finder
+// that bumped it would make every scan look like a read.
+// ---------------------------------------------------------------------------
+/** The three places a pattern is looked for. */
+export const GREP_FIELDS = ['text', 'metadata', 'file'];
+/** A scan is bounded, because a pattern may have come from a model. */
+const DEADLINE_MS = 2000;
+const DEADLINE_EVERY = 100;
+/** Past this a remembered file is reported as unsearched rather than read. */
+export const MAX_GREP_FILE_BYTES = 256 * 1024;
+/** And past this, so does the rest of the scan: one memory can hold hundreds. */
+export const MAX_GREP_BYTES = 8 * 1024 * 1024;
+/** Enough to see why a node matched without one file crowding out every other. */
+const HITS_PER_NODE = 5;
+/** A matched line, as much of it as is worth carrying. */
+const EXCERPT = 240;
+export async function grepMemory(store, pattern, filter = {}) {
+    const match = compile(pattern, filter);
+    const fields = new Set(filter.in?.length ? filter.in : GREP_FIELDS);
+    const kinds = filter.kinds?.length ? new Set(filter.kinds) : undefined;
+    const cap = filter.hitsPerNode ?? HITS_PER_NODE;
+    const want = filter.stale ?? 'exclude';
+    const superseded = store.graph.superseded(filter.sees);
+    const matches = [];
+    const skipped = [];
+    const until = Date.now() + DEADLINE_MS;
+    let budget = MAX_GREP_BYTES;
+    let seen = 0;
+    for (const node of store.graph.nodes(filter.sees)) {
+        if (++seen % DEADLINE_EVERY === 0 && Date.now() > until) {
+            throw new MemoryError(`the pattern is still running after ${DEADLINE_MS / 1000}s — it is too expensive to be useful`, 'anchor the pattern, or narrow the scan with `kinds` or `in`');
+        }
+        const stale = superseded.has(node.id);
+        if (want === 'exclude' ? stale : want === 'only' && !stale) {
+            continue;
+        }
+        if (kinds && !kinds.has(node.kind)) {
+            continue;
+        }
+        if (filter.audience && !node.audience.includes(filter.audience)) {
+            continue;
+        }
+        if (filter.files && !node.file) {
+            continue;
+        }
+        const hits = [];
+        let total = 0;
+        if (fields.has('text')) {
+            total += scan(hits, 'text', node.text, match, cap);
+        }
+        // Indented rather than compact, so a line number names a key someone
+        // can search for: one-line JSON would make every hit "line 1".
+        if (fields.has('metadata') && node.metadata) {
+            total += scan(hits, 'metadata', JSON.stringify(node.metadata, null, 4), match, cap);
+        }
+        if (fields.has('file') && node.file) {
+            const body = await read(store.dir, node.file, budget);
+            if (body.omitted) {
+                skipped.push({ id: node.id, path: node.file.path, reason: body.omitted });
+            }
+            else {
+                budget -= body.bytes;
+                total += scan(hits, 'file', body.text, match, cap);
+            }
+        }
+        if (hits.length) {
+            matches.push({ node, hits, more: total - hits.length, stale });
+        }
+    }
+    // Newest first, as `zen memory ls` lists them: when a pattern matches an
+    // old node and the node that corrected it, the correction should read first.
+    matches.sort((a, b) => b.node.createdAt.localeCompare(a.node.createdAt) || a.node.id.localeCompare(b.node.id));
+    const found = matches.length;
+    const limit = filter.limit;
+    const truncated = limit !== undefined && limit >= 0 && limit < found;
+    return { found, matches: truncated ? matches.slice(0, limit) : matches, truncated, skipped };
+}
+// ---------------------------------------------------------------------------
+/**
+ * A bad pattern is the model's mistake to fix, and `MemoryError` is the one
+ * shape both surfaces already know how to hand back rather than throw.
+ */
+function compile(pattern, options) {
+    try {
+        return matcher(pattern, options);
+    }
+    catch (err) {
+        if (err instanceof PatternError) {
+            throw new MemoryError(err.message, options.regex
+                ? 'fix the expression, or drop `regex` to search for the literal text'
+                : 'pass a shorter, non-empty pattern');
+        }
+        throw err;
+    }
+}
+/** Every matching line, of which the first `cap` are kept; the count is of all of them. */
+function scan(out, where, body, match, cap) {
+    const lines = body.split('\n');
+    let total = 0;
+    for (let i = 0; i < lines.length; i++) {
+        const at = match(lines[i]);
+        if (at < 0) {
+            continue;
+        }
+        total++;
+        if (out.length < cap) {
+            out.push({ where, line: i + 1, text: excerpt(lines[i], at) });
+        }
+    }
+    return total;
+}
+/**
+ * The line, windowed onto the match. Clipping from the front would be right for
+ * prose and useless for the one case that most needs it — a minified file or a
+ * long serialized value, where the hit is thousands of characters in.
+ */
+function excerpt(line, at) {
+    const text = line.replace(/\t/g, '    ').trimEnd();
+    if (text.length <= EXCERPT) {
+        return text;
+    }
+    const start = Math.max(0, Math.min(at - Math.floor(EXCERPT / 3), text.length - EXCERPT));
+    const end = start + EXCERPT;
+    return (start > 0 ? '…' : '') + text.slice(start, end) + (end < text.length ? '…' : '');
+}
+/**
+ * A remembered file's bytes, or why they were not read. The recorded size is
+ * not trusted for the decision — what is on disk is what would have to be
+ * scanned — and a skip is reported rather than swallowed, because a grep that
+ * silently did not look is worse than one that refused to.
+ */
+async function read(dir, file, budget) {
+    const path = hostPath(dir, file);
+    let size;
+    try {
+        size = (await stat(path)).size;
+    }
+    catch {
+        return { omitted: 'missing' };
+    }
+    if (size > Math.min(MAX_GREP_FILE_BYTES, budget)) {
+        return { omitted: 'too-big' };
+    }
+    let buf;
+    try {
+        buf = await readFile(path);
+    }
+    catch {
+        return { omitted: 'unreadable' };
+    }
+    // A NUL byte in the first few KiB is the cheap, boring test for "this is
+    // not text", and it is right about every format anyone remembers on purpose.
+    if (buf.subarray(0, 8192).includes(0)) {
+        return { omitted: 'binary' };
+    }
+    return { text: buf.toString('utf8'), bytes: buf.byteLength };
+}
+//# sourceMappingURL=grep.js.map

package/dist/memory/index.d.ts

@@ -73,8 +73,16 @@ export declare class MemoryIndex {
     get graph(): MemoryGraph;
     get kinds(): readonly string[];
     get relations(): readonly string[];
-    /** Read-only: a search returns nodes the model has not read, so nothing is touched. */
-    search(query: MemoryQuery, sees: readonly string[]): Promise<Recollection>;
+    /**
+     * Read-only: a search returns nodes the model has not read, so nothing is
+     * touched. `stale` is the audit view — it puts back the superseded nodes
+     * recall drops, which is the only way to see the correction and the thing
+     * it corrected at once. No tool passes it; it is for `zen memory search`,
+     * where the question is why a run recalled what it did.
+     */
+    search(query: MemoryQuery, sees: readonly string[], opts?: {
+        stale?: boolean;
+    }): Promise<Recollection>;
     /**
      * Listed rather than ranked, and deliberately not routed through `recall`:
      * ranking decays by `lastUsedAt`, only `memory_load` bumps that, and a
@@ -91,4 +99,6 @@ export declare class MemoryIndex {
     commit(tx: MemoryTransaction, opts: CommitOptions): Promise<CommitResult>;
     forget(ids: readonly string[], sees: readonly string[]): Promise<MemoryNode[]>;
 }
+/** Whitespace and case are not what makes two memories different. */
+export declare function flatten(text: string): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/memory/index.js

@@ -13,7 +13,7 @@ import { ALL_AGENTS, MEMORY_KINDS, MEMORY_RELATIONS, MemoryError, PREFERENCE_KIN
 // `ref` — it cannot know an id that has not been minted — and edges join refs
 // and existing ids indiscriminately, so a whole subgraph arrives in one call.
 // Everything is validated, and every source file stat-ed, before the first
-// mutation, on the same principle as `runPatch` in tools/workspace.ts: a
+// mutation, on the same principle as `runPatch` in tools/files.ts: a
 // commit that fails must leave no half-built graph behind.
 // ---------------------------------------------------------------------------
 /** Past this a `load` returns the path and lets the agent open it itself. */
@@ -44,8 +44,14 @@ export class MemoryIndex {
     get relations() {
         return [...this.#relations];
     }
-    /** Read-only: a search returns nodes the model has not read, so nothing is touched. */
-    async search(query, sees) {
+    /**
+     * Read-only: a search returns nodes the model has not read, so nothing is
+     * touched. `stale` is the audit view — it puts back the superseded nodes
+     * recall drops, which is the only way to see the correction and the thing
+     * it corrected at once. No tool passes it; it is for `zen memory search`,
+     * where the question is why a run recalled what it did.
+     */
+    async search(query, sees, opts = {}) {
         const vector = query.text ? await this.#embed([query.text], 'query') : undefined;
         return recall({
             graph: this.graph,
@@ -53,6 +59,7 @@ export class MemoryIndex {
             sees,
             vector: vector?.[0],
             vectors: this.store.vectors,
+            stale: opts.stale,
         });
     }
     /**
@@ -92,7 +99,11 @@ export class MemoryIndex {
                     : undefined,
             });
         }
-        await this.store.commit();
+        // The bump is bookkeeping, and persisting it is what would make a
+        // read-only recall rewrite the graph every other run is reading.
+        if (!this.store.readOnly) {
+            await this.store.commit();
+        }
         return out;
     }
     async commit(tx, opts) {
@@ -383,7 +394,7 @@ export class MemoryIndex {
     }
 }
 /** Whitespace and case are not what makes two memories different. */
-function flatten(text) {
+export function flatten(text) {
     return text.replace(/\s+/g, ' ').trim().toLowerCase();
 }
 //# sourceMappingURL=index.js.map

package/dist/memory/match.d.ts

@@ -0,0 +1,18 @@
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export declare const MAX_PATTERN = 200;
+export declare class PatternError extends Error {
+}
+export interface MatchOptions {
+    /** read the pattern as a regular expression rather than as a literal */
+    regex?: boolean;
+    caseSensitive?: boolean;
+}
+/** The offset of the first match in `text`, or -1. */
+export type Matcher = (text: string) => number;
+/**
+ * A finder over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
+//# sourceMappingURL=match.d.ts.map

package/dist/memory/match.js

@@ -0,0 +1,61 @@
+// ---------------------------------------------------------------------------
+// Matching, with nothing learned in between
+//
+// Everything here is exact. A substring is a substring, which is the whole
+// point of the surface built on it: the vector index answers "what is near
+// this", and near is a ranking, so it can only ever return the top of a list.
+// When the question is "does the word `password` appear anywhere at all", a
+// ranking is the wrong instrument and no amount of tuning makes it the right
+// one.
+//
+// This is a copy of `common/match.ts` in @zenera/rag, and has to be: the
+// dependency runs rag → cli → neo, so neo importing rag would close a cycle.
+// Two differences earn their place rather than being drift. A matcher returns
+// the offset it matched at, because grep reports *where*, not merely whether;
+// and there are no globs, because the haystack here is prose and a glob is
+// anchored to the whole string, so `error*` would match a line only when the
+// line began with the word.
+//
+// A pattern may arrive from a model, so a regex is a bounded promise: the
+// length is capped here and the scan that uses it keeps a deadline.
+// ---------------------------------------------------------------------------
+/** Long enough for any honest pattern, short enough to bound a bad one. */
+export const MAX_PATTERN = 200;
+export class PatternError extends Error {
+}
+/**
+ * A finder over a string. Literal by default: someone typing `user.id` means
+ * those seven characters, and a dot that quietly matched anything would be a
+ * worse answer than no answer.
+ */
+export function matcher(pattern, options = {}) {
+    guard(pattern);
+    if (!options.regex) {
+        if (options.caseSensitive) {
+            return (text) => text.indexOf(pattern);
+        }
+        const needle = pattern.toLowerCase();
+        return (text) => text.toLowerCase().indexOf(needle);
+    }
+    const expression = compile(pattern, options.caseSensitive ? '' : 'i');
+    // `lastIndex` is not carried between calls: the flags never include `g`.
+    return (text) => text.search(expression);
+}
+// ---------------------------------------------------------------------------
+function guard(pattern) {
+    if (pattern.length === 0) {
+        throw new PatternError('the pattern is empty');
+    }
+    if (pattern.length > MAX_PATTERN) {
+        throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
+    }
+}
+function compile(source, flags) {
+    try {
+        return new RegExp(source, flags);
+    }
+    catch (err) {
+        throw new PatternError(`invalid pattern: ${err.message}`);
+    }
+}
+//# sourceMappingURL=match.js.map

package/dist/memory/merge.d.ts

@@ -0,0 +1,65 @@
+import { MemoryStore, type MemoryEmbedding } from './store.ts';
+import { MemoryError } from './types.ts';
+export interface MergeOptions {
+    /** fold near-identical memories onto the one already there; on by default */
+    dedupe?: boolean;
+    /** resolve divergence by taking the highest revision instead of refusing */
+    force?: boolean;
+    /** plan and report, write nothing */
+    dryRun?: boolean;
+}
+export interface MergeSource {
+    dir: string;
+    /** what the source holds */
+    nodes: number;
+    edges: number;
+    /** what it contributed */
+    added: number;
+    /** ids the target already had, reconciled rather than duplicated */
+    shared: number;
+    /** new ids that turned out to be a memory the target already held */
+    twins: number;
+    files: number;
+}
+export interface MergeConflict {
+    id: string;
+    /** the source that disagreed */
+    dir: string;
+    /** revision already in the merge, and the one the source brought */
+    mine: number;
+    theirs: number;
+    text: string;
+}
+export interface MergeReport {
+    /** the memory everything was folded into */
+    dir: string;
+    sources: MergeSource[];
+    added: number;
+    shared: number;
+    twins: number;
+    /** edges the merge added, after re-pointing folded ends */
+    edges: number;
+    files: number;
+    embedding?: MemoryEmbedding;
+    dryRun: boolean;
+}
+/**
+ * Refusal carrying what to look at. Thrown before anything is written, so a
+ * conflicted merge leaves the target exactly as it was.
+ */
+export declare class MergeConflicts extends MemoryError {
+    readonly conflicts: readonly MergeConflict[];
+    constructor(conflicts: readonly MergeConflict[]);
+}
+/**
+ * Fold `sources` into `target`, left to right.
+ *
+ * Order is visible: a memory folded onto a twin keeps the id of whichever copy
+ * arrived first, so `merge a b` and `merge b a` can differ in which id
+ * survives. They never differ in what is remembered.
+ *
+ * On `dryRun` the target is left with the merge applied in memory and nothing
+ * on disk. Report the result and discard the store; do not commit it.
+ */
+export declare function mergeMemories(target: MemoryStore, sources: readonly string[], opts?: MergeOptions): Promise<MergeReport>;
+//# sourceMappingURL=merge.d.ts.map