memtrails 0.2.0 → 0.4.0

package/dist/memory.js

@@ -15,20 +15,20 @@
  * are written again from the logs.
  */
 import { join, resolve } from 'node:path';
-import { mkdirSync } from 'node:fs';
+import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
 import { Consolidator } from './core/consolidate.js';
-import { remoteEmbedder } from './core/embedder.js';
+import { remoteEmbedder, resolveEmbedRoute } from './core/embedder.js';
 import { memoryGather } from './core/gather.js';
 import { loadLocalEmbedder } from './core/local-embedder.js';
 import { ensureModel } from './core/model.js';
 import { ensureQdrant } from './core/qdrant-binary.js';
-import { COLLECTION, QdrantClient } from './core/qdrant-client.js';
-import { QdrantProcess } from './core/qdrant-process.js';
+import { COLLECTION, INDEX_LAYOUT, QdrantClient } from './core/qdrant-client.js';
+import { alive, bootTime, QdrantProcess, sameBoot } from './core/qdrant-process.js';
 import { memoryExplore, memoryOpen, memorySearch } from './core/recall.js';
 import { RecallBook } from './core/recall-log.js';
 import { episodeBody } from './core/episodes.js';
 import { MemoryRebuild } from './core/rebuild.js';
-import { remoteReranker } from './core/reranker.js';
+import { remoteReranker, resolveRerankRoute } from './core/reranker.js';
 import { sentenceRoom, sentenceSpans } from './core/room.js';
 import { Sleep } from './core/sleep.js';
 import { ltmLog, stmLog, UsageBook } from './core/store.js';
@@ -41,16 +41,133 @@ const BATCH = 32;
 const ROOM_TEXT = 1500;
 const ROOM_CHARS = 6000;
 const RECORD_LIMIT = 2_000_000;
+/** In the home: who has it open (lockHome), and the index layout its Qdrant collection was written in. */
+const LOCK_FILE = 'memory.lock';
+const LAYOUT_FILE = 'index.json';
+/** The homes this process holds, released if it exits without close(). */
+const holding = new Map();
+let exitHook = false;
+/** Who holds a lock: its live pid, 0 when it is being written this instant, null when nobody (stale or gone). */
+function lockHolder(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf8');
+    }
+    catch {
+        return { text: '', pid: null };
+    }
+    try {
+        const l = JSON.parse(text);
+        const pid = Number(l.pid);
+        return { text, pid: sameBoot(l.boot) && alive(pid) ? pid : null };
+    }
+    catch {
+        // Not written yet by the opener that just made it — or torn by a crash, and then old.
+        try {
+            return { text, pid: Date.now() - statSync(path).mtimeMs < 10_000 ? 0 : null };
+        }
+        catch {
+            return { text, pid: null };
+        }
+    }
+}
+/** One process at a time in a home: two would run two Qdrants on one storage and append to the logs at offsets the
+ *  other does not know. The lock is made exclusively (O_EXCL) with its pid and the boot; one whose process is gone, or
+ *  ran before a reboot (its pid may be another's now), is taken over. Returns its release. */
+function lockHome(dir) {
+    const path = join(dir, LOCK_FILE);
+    for (let attempt = 0;; attempt++) {
+        let fd;
+        try {
+            fd = openSync(path, 'wx', 0o600);
+        }
+        catch (err) {
+            if (err.code !== 'EEXIST')
+                throw err;
+            const { text, pid } = lockHolder(path);
+            if (pid !== null || attempt > 0) {
+                throw new Error(`memory ${dir} is open${pid ? ` in process ${pid}` : ''}: close it there first (if no process has it, delete ${path})`);
+            }
+            // Stale: taken over — unless another opener took it over since it was read.
+            try {
+                if (readFileSync(path, 'utf8') === text)
+                    rmSync(path, { force: true });
+            }
+            catch { /* gone already */ }
+            continue;
+        }
+        try {
+            writeSync(fd, JSON.stringify({ pid: process.pid, boot: bootTime() }));
+        }
+        finally {
+            closeSync(fd);
+        }
+        break;
+    }
+    const release = () => {
+        if (holding.get(path) !== release)
+            return;
+        holding.delete(path);
+        try {
+            if (JSON.parse(readFileSync(path, 'utf8')).pid === process.pid)
+                rmSync(path, { force: true });
+        }
+        catch { /* gone */ }
+    };
+    holding.set(path, release);
+    if (!exitHook) {
+        exitHook = true;
+        process.on('exit', () => { for (const r of [...holding.values()])
+            r(); });
+    }
+    return release;
+}
+/** The layout the index was written in (qdrant-client.ts INDEX_LAYOUT); null when not written down. */
+function indexLayout(dir) {
+    try {
+        return Number(JSON.parse(readFileSync(join(dir, LAYOUT_FILE), 'utf8')).layout);
+    }
+    catch {
+        return null;
+    }
+}
+function saveIndexLayout(dir) {
+    const path = join(dir, LAYOUT_FILE);
+    writeFileSync(`${path}.tmp`, JSON.stringify({ layout: INDEX_LAYOUT }), { mode: 0o600 });
+    renameSync(`${path}.tmp`, path);
+}
 export async function openMemory(o) {
     const dir = resolve(o.home);
-    mkdirSync(dir, { recursive: true });
     const log = o.log ?? (() => { });
-    // Embedding: here, or on a server.
+    // Private: the logs, the index and the word vectors are everything that was added (a shared machine).
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    try {
+        chmodSync(dir, 0o700);
+    }
+    catch (err) {
+        log(`[memtrails] ${dir} could not be made private: ${err.message}`);
+    }
+    const release = lockHome(dir);
+    const started = { proc: null, local: null };
+    try {
+        return await openIn(dir, o, log, started, release);
+    }
+    catch (err) {
+        await started.proc?.stopAndWait();
+        await started.local?.close().catch(() => undefined);
+        release();
+        throw err;
+    }
+}
+async function openIn(dir, o, log, started, release) {
+    // Embedding: here, or on a server — the options' own, else the one the environment names.
     const e = o.embedding ?? {};
-    const local = !e.server || e.modelDir
+    const server = e.server ?? (e.modelDir ? null : resolveEmbedRoute());
+    const local = !server || e.modelDir
         ? await loadLocalEmbedder(e.modelDir ?? await ensureModel({ log }), { threads: e.threads ?? 2 })
         : null;
-    const embed = e.server ? remoteEmbedder({ route: () => e.server }) : local.embed;
+    started.local = local;
+    const embed = server ? remoteEmbedder({ route: () => server }) : local.embed;
     const encode = local?.words ?? null;
     const words = new WordStore(join(dir, 'words.bin'));
     await words.load();
@@ -64,6 +181,7 @@ export async function openMemory(o) {
     }
     else {
         proc = new QdrantProcess({ binary: q && 'binary' in q ? q.binary : await ensureQdrant({ log }), dir: join(dir, 'qdrant'), log });
+        started.proc = proc;
         await proc.start();
         endpoint = () => proc.current();
     }
@@ -75,24 +193,56 @@ export async function openMemory(o) {
     await ltm.load();
     const usage = new UsageBook(join(dir, 'usage.json'));
     const book = new RecallBook(join(dir, 'recall.jsonl'));
+    void book.trim(); // the documented cap: once an open, and every so many events while open (recall-log.ts)
     const ask = typeof o.llm === 'function' ? o.llm : o.llm ? openAiAsk(o.llm) : defaultAsk;
-    const base = { client, embed, stm, ltm, log, aborted: () => false, ask };
+    /** Set by close(): what runs stops at its next step. */
+    let closing = false;
+    const aborted = () => closing;
+    const base = { client, embed, stm, ltm, log, aborted, ask };
+    const rerankEnv = o.rerank === undefined ? resolveRerankRoute() : null;
     const rerank = o.rerank === false ? undefined
         : typeof o.rerank === 'function' ? o.rerank
             : o.rerank && typeof o.rerank === 'object' ? remoteReranker({ route: () => o.rerank })
-                : encode ? wordReranker(encode, words) : undefined;
+                : rerankEnv ? remoteReranker({ route: () => rerankEnv })
+                    : encode ? wordReranker(encode, words) : undefined;
     const deps = { client, embed, rerank, used: (ids, what) => usage.note(ids, what), record: (ev) => book.record(ev) };
-    // The index is derived: empty while the logs are not (a new Qdrant, a deleted folder), it is written again.
+    // The index is derived: written again from the logs when it is empty while they are not (a new Qdrant, a deleted
+    // folder), or was written in another layout — or in one not written down (an index from before the layouts were).
     const rebuild = new MemoryRebuild({
-        path: join(dir, 'rebuild.json'), stm, ltm, aborted: () => false,
+        path: join(dir, 'rebuild.json'), stm, ltm, aborted,
         writeStm: (xs) => writeStm(base, xs), writeLtm: (xs) => writeLtm(base, xs),
     });
-    if (rebuild.pending() || (!(await client.count()) && (stm.size() || ltm.size()))) {
+    const indexed = await client.count();
+    const layout = indexLayout(dir);
+    if (layout !== INDEX_LAYOUT) {
+        if (indexed) {
+            log(`[memtrails] the index was written in layout ${layout ?? 'unknown'}, not ${INDEX_LAYOUT}: writing it again from the logs`);
+            rebuild.begin(); // before the drop: a crash after it resumes
+            await client.dropCollection();
+            await client.ensureCollection();
+        }
+        saveIndexLayout(dir);
+    }
+    if (rebuild.pending() || (!indexed && (stm.size() || ltm.size()))) {
         log('[memtrails] writing the index from the logs');
         if (!rebuild.pending())
             rebuild.begin();
         await rebuild.run();
     }
+    // consolidate, sleep, rebuild, indexWords and remove one at a time: each reads the logs and writes the index from
+    // them — two consolidations pay the model twice and write stale copies over each other, a rebuild clears what
+    // another is writing, a remove during a consolidation would see its record's episode written back after it.
+    let queue = Promise.resolve();
+    const exclusive = (job) => {
+        const run = queue.then(() => {
+            if (closing)
+                throw new Error('memory is closed');
+            return job();
+        });
+        queue = run.catch(() => undefined);
+        return run;
+    };
+    let closed = null;
     /** The text word by word scores an item by: its key, as search returns it. */
     const keyText = (title, summary) => `${title}\n${summary}`;
     /** A found item's whole text for the room: a record's own, or what consolidation wrote. */
@@ -110,7 +260,7 @@ export async function openMemory(o) {
         if (!encode)
             return 0;
         const todo = [...new Set(texts)].filter((t) => t.trim() && !words.get(t));
-        for (let i = 0; i < todo.length; i += BATCH) {
+        for (let i = 0; i < todo.length && !closing; i += BATCH) {
             const part = todo.slice(i, i + BATCH);
             const encoded = await encode(part, 'document');
             await words.put(part.map((t, n) => [t, packWords(encoded[n])]));
@@ -132,6 +282,22 @@ export async function openMemory(o) {
         stm.compact();
         return { added: fresh.length, unchanged: total - fresh.length };
     };
+    /** Unlocked: consolidate and sleep run it inside their own turn. */
+    const indexWords = async () => {
+        if (!encode)
+            return 0;
+        const texts = [...stm.all()].flatMap((x) => wordsOfEntries([x]));
+        for (const [uid, m] of ltm.entries()) {
+            if (m.status !== 'active')
+                continue;
+            const it = ltm.get(uid);
+            if (!it)
+                continue;
+            const room = `${it.title}\n${it.summary}\n${it.body}`.slice(0, ROOM_TEXT);
+            texts.push(keyText(it.title, it.summary), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b)));
+        }
+        return encodeMissing(texts);
+    };
     const memory = {
         async add(records) {
             const list = Array.isArray(records) ? records : [records];
@@ -172,14 +338,19 @@ export async function openMemory(o) {
             await flush();
             return total;
         },
-        async remove(ids) {
+        remove: (ids) => exclusive(async () => {
             const uids = ids.map((x) => recordUid(typeof x === 'string' ? { id: x } : x)).filter((u) => stm.meta(u));
             if (!uids.length)
                 return 0;
-            await client.deleteItems(uids.flatMap((u) => [stmItem(u), `ep:${u}`]));
+            const episodes = uids.map((u) => `ep:${u}`);
+            await client.deleteItems([...uids.map(stmItem), ...episodes]);
             stm.remove(uids);
+            // A kept episode is the record's own text: gone from the long-term log too, or a rebuild indexes it again and the
+            // profiles go on reading it.
+            ltm.remove(episodes);
+            await Promise.all([stm.flush(), ltm.flush()]);
             return uids.length;
-        },
+        }),
         search: (s) => memorySearch(deps, s),
         searchText: async (s) => searchText(await memorySearch(deps, s), s.order ?? 'relevance'),
         async room(s) {
@@ -201,11 +372,11 @@ export async function openMemory(o) {
             return sources.length ? { ...p, sources } : p;
         },
         explore: (s) => memoryExplore(deps, s),
-        gather: (s) => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}) }),
-        async consolidate() {
+        gather: (s, g) => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}), ...(g?.notes ? { notes: true } : {}) }),
+        consolidate: () => exclusive(async () => {
             const worker = new Consolidator({ ...base, concurrency: o.concurrency ?? 4 });
             const total = { windows: 0, added: 0, merged: 0, updated: 0, superseded: 0, linked: 0, dropped: 0, episodes: 0, observed: 0, cued: 0 };
-            for (let round = 0; round < 1000 && worker.pending().length; round++) {
+            for (let round = 0; round < 1000 && !closing && worker.pending().length; round++) {
                 const r = await worker.run(64);
                 for (const k of Object.keys(total))
                     if (typeof r[k] === 'number')
@@ -217,45 +388,42 @@ export async function openMemory(o) {
                 if (!r.windows)
                     break;
             }
-            await memory.indexWords();
+            await indexWords();
             return total;
-        },
-        async sleep() {
+        }),
+        sleep: () => exclusive(async () => {
             const r = await new Sleep({ ...base, usage, statePath: join(dir, 'sleep.json') }).run();
-            await memory.indexWords();
+            await indexWords();
             return r;
-        },
-        async indexWords() {
-            if (!encode)
-                return 0;
-            const texts = [...stm.all()].flatMap((x) => wordsOfEntries([x]));
-            for (const [uid, m] of ltm.entries()) {
-                if (m.status !== 'active')
-                    continue;
-                const it = ltm.get(uid);
-                if (!it)
-                    continue;
-                const room = `${it.title}\n${it.summary}\n${it.body}`.slice(0, ROOM_TEXT);
-                texts.push(keyText(it.title, it.summary), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b)));
-            }
-            return encodeMissing(texts);
-        },
-        async rebuild() {
+        }),
+        indexWords: () => exclusive(indexWords),
+        rebuild: () => exclusive(async () => {
+            rebuild.begin(); // before the drop: a crash, or a close, after it resumes at the next open
             await client.dropCollection();
             await client.ensureCollection();
-            rebuild.begin();
             await rebuild.run();
-        },
+        }),
         tools: () => buildAgentTools({
             search: (s) => memorySearch(deps, s),
             open: (id) => memory.open(id),
             explore: (s) => memoryExplore(deps, s),
         }),
         stats: () => ({ records: stm.size(), items: [...ltm.entries()].filter(([, m]) => m.status === 'active').length, words: words.size }),
-        async close() {
-            usage.flush();
-            await proc?.stopAndWait();
-            await local?.close();
+        close() {
+            closed ??= (async () => {
+                closing = true;
+                await queue; // the job running stops at its next step; those waiting are refused
+                try {
+                    await book.settled();
+                    usage.flush();
+                    await proc?.stopAndWait();
+                    await local?.close();
+                }
+                finally {
+                    release();
+                }
+            })();
+            return closed;
         },
     };
     return memory;

package/dist/records.js

@@ -70,7 +70,7 @@ export function entryOf(r, now, turn = 0) {
  * Turns before the first request (an assistant that opened) are the first record's own.
  */
 export function conversationEntries(c, now) {
-    const start = timeOf(c.at) ?? now;
+    const began = timeOf(c.at);
     const out = [];
     let request = [];
     let reply = [];
@@ -79,10 +79,13 @@ export function conversationEntries(c, now) {
         if (!request.length && !reply.length)
             return;
         const n = out.length;
+        // A time only the caller gave is part of the fingerprint: one defaulted from when it is added would make the same
+        // conversation added again a changed one — read, embedded and consolidated again.
+        const given = at ?? (began === undefined ? undefined : began + n * 60_000);
         const e = entryOf({ id: `${c.id}#${n}`, source: c.source, collection: c.collection, thread: c.id, kind: 'conversation',
             title: request.join('\n').split('\n').find((l) => l.trim())?.trim() || reply.join('\n').split('\n').find((l) => l.trim())?.trim() || c.id,
-            text: request.join('\n'), at: at ?? start + n * 60_000 }, now, n);
-        out.push({ ...e, request: request.join('\n') || e.title, reply: reply.join('\n'), said: reply.join('\n') });
+            text: request.join('\n'), at: given }, now, n);
+        out.push({ ...e, ts: given ?? now + n * 60_000, request: request.join('\n') || e.title, reply: reply.join('\n'), said: reply.join('\n') });
         request = [];
         reply = [];
         at = undefined;

package/dist/tools/agent-tools.d.ts

@@ -33,7 +33,6 @@ export interface ToolProvider {
         limit?: number;
     }): Promise<Explored>;
 }
-type CrossMemoryHit = RecalledItem;
 /** A date as a reader reckons with it — the weekday too ("the Friday before …"): 2023-07-14 → "Fri 14 Jul 2023". */
 export declare function readableDate(d: string): string;
 /** How long something lasted, as a reader reckons with it: "since Feb 2022", "Nov 2021 – May 2022". */
@@ -41,12 +40,10 @@ export declare function lastedText(h: {
     startAt?: unknown;
     endAt?: unknown;
 }): string;
-/** All of a profile — what memory knows about one subject, one dated fact a line (core/profiles.ts) — up to this:
- *  measured, a profile cut like any other item lost most of its gain. */
-export declare const PROFILE_SHOWN = 1500;
 /** What memory_search shows: an observation as one dated line (when it happened, and when it was told when that
  *  differs), a profile whole (up to PROFILE_SHOWN), anything else as its title line and summary — shorter when many
  *  are asked for. Best first, or by time. */
-export declare function searchText(hits: CrossMemoryHit[], order?: 'relevance' | 'time'): string;
+export declare function searchText(hits: RecalledItem[], order?: 'relevance' | 'time'): string;
+/** memory_search, memory_open and memory_explore (when the provider explores) over a memory. */
 export declare function buildAgentTools(p: ToolProvider): AgentTool[];
 export {};

package/dist/tools/agent-tools.js

@@ -6,8 +6,19 @@
  *   const tools = memory.tools();   // hand them to an MCP server, or to a model's tool calling
  */
 import { z } from 'zod';
+import { PROFILE_SHOWN } from '../core/profiles.js';
+import { cut } from '../core/text.js';
+import { dateSpan } from '../core/writer.js';
 function tool(name, description, inputSchema, handler) {
-    return { name, description, inputSchema, jsonSchema: z.toJSONSchema(z.object(inputSchema)), handler: handler };
+    const schema = z.object(inputSchema);
+    // Checked here too: a caller doing function calling hands the model's arguments straight to the handler.
+    const checked = async (args, extra) => {
+        const got = schema.safeParse(args ?? {});
+        if (!got.success)
+            return text(`Invalid arguments: ${got.error.issues.map((i) => `${i.path.join('.') || 'input'}: ${i.message}`).join('; ')}`, true);
+        return handler(got.data, extra);
+    };
+    return { name, description, inputSchema, jsonSchema: z.toJSONSchema(schema), handler: checked };
 }
 const DESCRIPTIONS = {
     memory_search: 'Search memory: everything it was given (e-mails, messages, notes, documents, conversations, commits, …) and '
@@ -20,12 +31,18 @@ const DESCRIPTIONS = {
         + 'people, places, systems, names — and links between memories, to what a search for the words does not find. Returns '
         + 'what search found, what is related, and the concepts around it to explore next.',
 };
+/** What a model may send: a question, a topic or an id is a line, not a document; a few dozen collections at most. */
+const TEXT_MAX = 2000;
+const COLLECTIONS_MAX = 50;
 const day = (ts) => (ts ? new Date(ts).toISOString().slice(0, 10) : '');
 const hitLine = (h, n) => `${n + 1}. ${h.title} (${h.kind === 'episode' ? 'earlier work' : h.kind} · ${day(h.ts)}${h.record?.author ? ` · ${h.record.author}` : ''} · id ${h.id})`;
 const WEEKDAY = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
 const MONTH = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
 /** A date as a reader reckons with it — the weekday too ("the Friday before …"): 2023-07-14 → "Fri 14 Jul 2023". */
 export function readableDate(d) {
+    // Not a date (2023-13, 2023-02-30): shown as written rather than as "undefined 2023" or a day that does not exist.
+    if (!dateSpan(d))
+        return d;
     const [y, m, dd] = d.split('-').map(Number);
     if (!y)
         return '';
@@ -44,10 +61,6 @@ export function lastedText(h) {
 }
 /** When a hit happened, for ordering by time: an observation's own date, else when it was learned. */
 const when = (h) => h.eventAt || day(h.ts);
-/** All of a profile — what memory knows about one subject, one dated fact a line (core/profiles.ts) — up to this:
- *  measured, a profile cut like any other item lost most of its gain. */
-export const PROFILE_SHOWN = 1500;
-const cutAt = (s, n) => (s.length > n ? `${s.slice(0, n)}…` : s);
 /** What memory_search shows: an observation as one dated line (when it happened, and when it was told when that
  *  differs), a profile whole (up to PROFILE_SHOWN), anything else as its title line and summary — shorter when many
  *  are asked for. Best first, or by time. */
@@ -57,7 +70,7 @@ export function searchText(hits, order = 'relevance') {
     const compact = list.length > 20;
     const lines = list.map((h, n) => {
         if (h.kind === 'profile')
-            return `${hitLine(h, n)}\n   ${cutAt(h.summary, PROFILE_SHOWN).replace(/\n/g, '\n   ')}`;
+            return `${hitLine(h, n)}\n   ${cut(h.summary, PROFILE_SHOWN).replace(/\n/g, '\n   ')}`;
         if (h.kind !== 'observation') {
             // The passage that matched, when it was not the start: a fact said deep in a long request is what was found.
             const matched = h.match ? ` … ${h.match.replace(/\s+/g, ' ')}` : '';
@@ -74,11 +87,12 @@ export function searchText(hits, order = 'relevance') {
     });
     return `${order === 'time' ? 'Oldest first — the date is when it happened (and when it was told, when that differs).\n' : ''}${lines.join('\n')}`;
 }
-function memoryTools(p) {
+/** memory_search, memory_open and memory_explore (when the provider explores) over a memory. */
+export function buildAgentTools(p) {
     return [
         tool('memory_search', DESCRIPTIONS.memory_search, {
-            query: z.string().describe('What you need to know, in plain words.'),
-            collections: z.array(z.string()).optional().describe('Only these collections (default: all).'),
+            query: z.string().max(TEXT_MAX).describe('What you need to know, in plain words.'),
+            collections: z.array(z.string().max(TEXT_MAX)).max(COLLECTIONS_MAX).optional().describe('Only these collections (default: all).'),
             limit: z.number().int().min(1).max(200).optional().describe('How many of each to return — dated observations, and other memories (default 20; 100 to 200 when you gather everything on a topic, list or count things, or need their dates).'),
             order: z.enum(['relevance', 'time']).optional().describe('Best first (default), or oldest first — for what happened when, and in what order.'),
             from: z.string().regex(/^\d{4}(-\d{2}(-\d{2})?)?$/).optional().describe('Only what happened on or after this date (YYYY-MM-DD, YYYY-MM or YYYY) — when the request names a time ("in June 2023", "last week").'),
@@ -94,25 +108,30 @@ function memoryTools(p) {
                 return text(err.message, true);
             }
         }),
-        tool('memory_open', DESCRIPTIONS.memory_open, { id: z.string().describe('The memory id, as memory_search or memory_explore gave it.') }, async (args) => {
-            const it = await p.open(args.id).catch(() => null);
+        tool('memory_open', DESCRIPTIONS.memory_open, { id: z.string().max(TEXT_MAX).describe('The memory id, as memory_search or memory_explore gave it.') }, async (args) => {
+            // An index that failed is not an id that is missing: the agent should not conclude the memory is gone.
+            let it;
+            try {
+                it = await p.open(args.id);
+            }
+            catch (err) {
+                return text(`Memory could not be read: ${err.message}`, true);
+            }
             if (!it)
                 return text(`There is no memory "${args.id}" (it may have been merged or cleared).`, true);
             const files = Array.isArray(it.files) && it.files.length ? `\nFiles: ${it.files.join(', ')}` : '';
             const told = day(Number(it.ts ?? 0));
             const happened = [typeof it.eventAt === 'string' && it.eventAt ? readableDate(it.eventAt) : '', lastedText(it)].filter(Boolean).join(', ');
             const when = happened ? `happened ${happened}; told ${readableDate(told)}` : told;
-            // The raw requests it was learned from, whole (the truth it is a key to), and the texts a merge folded into it.
+            // The raw requests it was learned from, whole (the truth it is a key to).
             const sources = Array.isArray(it.sources) && it.sources.length
                 ? `\n\n## Where it came from\n${it.sources.map((x) => `### ${day(x.when)} · ${x.title} (id ${x.id})\n${x.body}`).join('\n\n')}` : '';
-            const absorbed = Array.isArray(it.absorbed) && it.absorbed.length
-                ? `\n\n## Texts merged into it\n${it.absorbed.map((x) => `### ${x.title}\n${x.summary}\n${x.body}`).join('\n\n')}` : '';
-            return text(`# ${String(it.title ?? '')}\n(${String(it.kind ?? '')} · ${when} · ${String(it.project ?? '')})\n\n${String(it.summary ?? '')}\n\n${String(it.body ?? '')}${files}${sources}${absorbed}`);
+            return text(`# ${String(it.title ?? '')}\n(${String(it.kind ?? '')} · ${when} · ${String(it.project ?? '')})\n\n${String(it.summary ?? '')}\n\n${String(it.body ?? '')}${files}${sources}`);
         }),
         ...(p.explore ? [tool('memory_explore', DESCRIPTIONS.memory_explore, {
-                topic: z.string().optional().describe('What to explore, in plain words — a system, a project, a problem, a name.'),
-                from: z.array(z.string()).max(5).optional().describe('Memory ids to explore from, as memory_search or memory_explore gave them.'),
-                collections: z.array(z.string()).optional().describe('Only these collections (default: all).'),
+                topic: z.string().max(TEXT_MAX).optional().describe('What to explore, in plain words — a system, a project, a problem, a name.'),
+                from: z.array(z.string().max(TEXT_MAX)).max(5).optional().describe('Memory ids to explore from, as memory_search or memory_explore gave them.'),
+                collections: z.array(z.string().max(TEXT_MAX)).max(COLLECTIONS_MAX).optional().describe('Only these collections (default: all).'),
                 limit: z.number().int().min(1).max(20).optional().describe('How many related items to return (default 10).'),
             }, async (args) => {
                 if (!args.topic?.trim() && !args.from?.length)
@@ -137,6 +156,3 @@ function memoryTools(p) {
     ];
 }
 const text = (t, isError = false) => ({ content: [{ type: 'text', text: t }], ...(isError ? { isError: true } : {}) });
-export function buildAgentTools(p) {
-    return memoryTools(p);
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memtrails",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Retrieval and exploration for any text — e-mail, chat, notes, documents, news, commits, conversations. Searchable at once, on your own machine, in any language; optional consolidation into typed knowledge with a language model.",
   "keywords": [
     "retrieval",