memtrails 0.3.0 → 0.4.1

package/dist/llm.js

@@ -5,34 +5,77 @@
  * `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
  * any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
  */
+import { redactString } from './core/redact.js';
+const OPENAI = 'https://api.openai.com/v1';
 const why = (reason, req) => `${reason}${req.consequence ? `, ${req.consequence}` : ''}`;
+/** A wait that an abort ends early. */
+const pause = (ms, signal) => new Promise((ok) => {
+    if (signal?.aborted)
+        return ok();
+    const done = () => { clearTimeout(t); signal?.removeEventListener('abort', done); ok(); };
+    const t = setTimeout(done, ms);
+    signal?.addEventListener('abort', done, { once: true });
+});
+/**
+ * `Promise.all` for asks side by side, each handed a signal: the first that throws aborts the others — an ask that
+ * honours `signal` stops, and none is paid for an answer no one reads — and they have all settled when it is thrown.
+ * `outer` (the caller's) aborts them too.
+ */
+export async function allOrAbort(jobs, outer) {
+    const c = new AbortController();
+    const follow = () => c.abort(outer?.reason);
+    if (outer?.aborted)
+        follow();
+    else
+        outer?.addEventListener('abort', follow, { once: true });
+    // Each started in order, at once, as Promise.all over async functions would.
+    const runs = jobs.map(async (job) => job(c.signal));
+    try {
+        return await Promise.all(runs);
+    }
+    catch (err) {
+        c.abort(err);
+        await Promise.allSettled(runs);
+        throw err;
+    }
+    finally {
+        outer?.removeEventListener('abort', follow);
+    }
+}
 export function openAiAsk(o = {}) {
     return async (req) => {
         const model = req.model ?? o.model ?? process.env.MEMTRAILS_MODEL?.trim();
-        const key = o.apiKey ?? (process.env.MEMTRAILS_LLM_API_KEY?.trim() || process.env.OPENAI_API_KEY?.trim());
+        const base = (o.baseUrl ?? (process.env.MEMTRAILS_LLM_BASE_URL?.trim() || OPENAI)).replace(/\/+$/, '');
+        // An OpenAI key goes to OpenAI only: with another address set, it would hand the key to whoever serves that.
+        const key = o.apiKey ?? (process.env.MEMTRAILS_LLM_API_KEY?.trim() || (base === OPENAI ? process.env.OPENAI_API_KEY?.trim() : undefined));
         if (!model)
             return { unavailable: why('no model is set (the model option, or MEMTRAILS_MODEL)', req) };
         if (!key)
-            return { unavailable: why('no API key is set (the apiKey option, MEMTRAILS_LLM_API_KEY or OPENAI_API_KEY)', req) };
-        const base = (o.baseUrl ?? (process.env.MEMTRAILS_LLM_BASE_URL?.trim() || 'https://api.openai.com/v1')).replace(/\/+$/, '');
+            return { unavailable: why(`no API key is set (the apiKey option or MEMTRAILS_LLM_API_KEY${base === OPENAI ? ', or OPENAI_API_KEY' : ''})`, req) };
         const body = JSON.stringify({
             model, instructions: req.system, input: req.prompt, max_output_tokens: req.maxTokens ?? 12_000,
             ...(o.reasoningEffort ? { reasoning: { effort: o.reasoningEffort } } : {}),
             text: { format: { type: 'json_schema', name: req.name ?? 'memory', strict: true, schema: req.schema } },
         });
         const tries = Math.max(1, (o.retries ?? 5) + 1);
+        const stopped = { unavailable: why('the ask was stopped', req) };
         let last = '';
         for (let attempt = 0; attempt < tries; attempt++) {
             if (attempt)
-                await new Promise((ok) => setTimeout(ok, Math.min(60_000, 1000 * 2 ** (attempt - 1)) * (0.5 + Math.random())));
+                await pause(Math.min(60_000, 1000 * 2 ** (attempt - 1)) * (0.5 + Math.random()), req.signal);
+            if (req.signal?.aborted)
+                return stopped;
+            const timeout = AbortSignal.timeout(req.timeoutMs ?? 180_000);
             let res;
             try {
                 res = await (o.fetchImpl ?? fetch)(`${base}/responses`, {
-                    method: 'POST', body, signal: AbortSignal.timeout(req.timeoutMs ?? 180_000),
+                    method: 'POST', body, signal: req.signal ? AbortSignal.any([req.signal, timeout]) : timeout,
                     headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
                 });
             }
             catch (err) {
+                if (req.signal?.aborted)
+                    return stopped;
                 last = err.message || err.name;
                 continue;
             }
@@ -40,8 +83,11 @@ export function openAiAsk(o = {}) {
                 last = `HTTP ${res.status}`;
                 continue;
             }
-            if (!res.ok)
-                return { unavailable: why(`the model answered HTTP ${res.status}`, req) };
+            if (!res.ok) {
+                // What the provider said (a wrong model name, a schema it refuses) — not only its status.
+                const said = redactString((await res.text().catch(() => '')).replace(/\s+/g, ' ').trim()).slice(0, 300);
+                return { unavailable: why(`the model answered HTTP ${res.status}${said ? `: ${said}` : ''}`, req) };
+            }
             try {
                 const r = (await res.json());
                 o.onUsage?.({ model: r.model ?? model, name: req.name, inputTokens: r.usage?.input_tokens ?? 0,

package/dist/memory.d.ts

@@ -14,14 +14,16 @@ export interface MemoryOptions {
      *  - default: on this machine, the model fetched on first use into the cache (218 MB);
      *  - `{ modelDir }`: on this machine, from a directory you provide;
      *  - `{ server }`: a server that serves it over the OpenAI embeddings API (text-embeddings-inference, Ollama, …).
-     *    Word by word then needs `{ modelDir }` too, or it is off. */
+     *    Word by word then needs `{ modelDir }` too, or it is off.
+     *  With neither `server` nor `modelDir`, MEMTRAILS_EMBED_URL (+ _KEY, _MODEL) names a server when it is set. */
     embedding?: {
         modelDir?: string;
         server?: EmbedRoute;
         threads?: number;
     };
     /** How the best 30 of a search are reordered: `'words'` (default with a local model: word by word, no server), a
-     *  cross-encoder server (`RerankRoute`), your own function, or `false` for the search's own order. */
+     *  cross-encoder server (`RerankRoute`), your own function, or `false` for the search's own order. Not given:
+     *  MEMTRAILS_RERANK_URL (+ _KEY, _MODEL) names a server when it is set. */
     rerank?: 'words' | RerankRoute | RecallDeps['rerank'] | false;
     /** The language model for consolidate, sleep and gather: your own `Ask`, or options for the OpenAI-compatible one.
      *  Default: the one the environment names (MEMTRAILS_MODEL and a key). */
@@ -103,6 +105,9 @@ export interface Memory {
         items: number;
         words: number;
     };
+    /** Stops a consolidate, sleep, rebuild or indexWords at its next step (a rebuild resumes at the next open) and waits
+     *  for it and for the adds and searches in flight, then releases Qdrant, the model and the home. Every call after it
+     *  is refused ("memory is closed"). */
     close(): Promise<void>;
 }
 export declare function openMemory(o: MemoryOptions): Promise<Memory>;

package/dist/memory.js

@@ -15,20 +15,20 @@
  * are written again from the logs.
  */
 import { join, resolve } from 'node:path';
-import { mkdirSync } from 'node:fs';
+import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
 import { Consolidator } from './core/consolidate.js';
-import { remoteEmbedder } from './core/embedder.js';
+import { remoteEmbedder, resolveEmbedRoute } from './core/embedder.js';
 import { memoryGather } from './core/gather.js';
 import { loadLocalEmbedder } from './core/local-embedder.js';
 import { ensureModel } from './core/model.js';
 import { ensureQdrant } from './core/qdrant-binary.js';
-import { COLLECTION, QdrantClient } from './core/qdrant-client.js';
-import { QdrantProcess } from './core/qdrant-process.js';
+import { COLLECTION, INDEX_LAYOUT, QdrantClient } from './core/qdrant-client.js';
+import { alive, bootTime, QdrantProcess, sameBoot } from './core/qdrant-process.js';
 import { memoryExplore, memoryOpen, memorySearch } from './core/recall.js';
 import { RecallBook } from './core/recall-log.js';
 import { episodeBody } from './core/episodes.js';
 import { MemoryRebuild } from './core/rebuild.js';
-import { remoteReranker } from './core/reranker.js';
+import { remoteReranker, resolveRerankRoute } from './core/reranker.js';
 import { sentenceRoom, sentenceSpans } from './core/room.js';
 import { Sleep } from './core/sleep.js';
 import { ltmLog, stmLog, UsageBook } from './core/store.js';
@@ -41,16 +41,171 @@ const BATCH = 32;
 const ROOM_TEXT = 1500;
 const ROOM_CHARS = 6000;
 const RECORD_LIMIT = 2_000_000;
+/** In the home: who has it open (lockHome), and the index layout its Qdrant collection was written in. */
+const LOCK_FILE = 'memory.lock';
+const LAYOUT_FILE = 'index.json';
+/** The homes this process holds, released if it exits without close(). */
+const holding = new Map();
+let exitHook = false;
+/** Who holds a lock: its live pid, 0 when it is being written this instant, null when nobody (stale or gone). */
+function lockHolder(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf8');
+    }
+    catch {
+        return { text: '', pid: null };
+    }
+    try {
+        const l = JSON.parse(text);
+        const pid = Number(l.pid);
+        return { text, pid: sameBoot(l.boot) && alive(pid) ? pid : null };
+    }
+    catch {
+        // Not written yet by the opener that just made it — or torn by a crash, and then old.
+        try {
+            return { text, pid: Date.now() - statSync(path).mtimeMs < 10_000 ? 0 : null };
+        }
+        catch {
+            return { text, pid: null };
+        }
+    }
+}
+/** One process at a time in a home: two would run two Qdrants on one storage and append to the logs at offsets the
+ *  other does not know. The lock is made exclusively (O_EXCL) with its pid and the boot; one whose process is gone, or
+ *  ran before a reboot (its pid may be another's now), is taken over. Returns its release. */
+function lockHome(dir) {
+    const path = join(dir, LOCK_FILE);
+    for (let attempt = 0;; attempt++) {
+        let fd;
+        try {
+            fd = openSync(path, 'wx', 0o600);
+        }
+        catch (err) {
+            if (err.code !== 'EEXIST')
+                throw err;
+            const { text, pid } = lockHolder(path);
+            if (pid !== null || attempt > 0) {
+                throw new Error(`memory ${dir} is open${pid ? ` in process ${pid}` : ''}: close it there first (if no process has it, delete ${path})`);
+            }
+            // Stale: taken over — unless another opener took it over since it was read.
+            try {
+                if (readFileSync(path, 'utf8') === text)
+                    rmSync(path, { force: true });
+            }
+            catch { /* gone already */ }
+            continue;
+        }
+        try {
+            writeSync(fd, JSON.stringify({ pid: process.pid, boot: bootTime() }));
+        }
+        finally {
+            closeSync(fd);
+        }
+        break;
+    }
+    const release = () => {
+        if (holding.get(path) !== release)
+            return;
+        holding.delete(path);
+        try {
+            if (JSON.parse(readFileSync(path, 'utf8')).pid === process.pid)
+                rmSync(path, { force: true });
+        }
+        catch { /* gone */ }
+    };
+    holding.set(path, release);
+    if (!exitHook) {
+        exitHook = true;
+        process.on('exit', () => { for (const r of [...holding.values()])
+            r(); });
+    }
+    return release;
+}
+/** The layout the index was written in (qdrant-client.ts INDEX_LAYOUT); null when not written down. */
+function indexLayout(dir) {
+    try {
+        return Number(JSON.parse(readFileSync(join(dir, LAYOUT_FILE), 'utf8')).layout);
+    }
+    catch {
+        return null;
+    }
+}
+function saveIndexLayout(dir) {
+    const path = join(dir, LAYOUT_FILE);
+    writeFileSync(`${path}.tmp`, JSON.stringify({ layout: INDEX_LAYOUT }), { mode: 0o600 });
+    renameSync(`${path}.tmp`, path);
+}
+/** Turns shared, and a turn alone: many share at once; one alone waits until no one shares, and keeps new sharers out
+ *  while it waits, so a stream of them never holds it off. */
+function turns() {
+    let sharing = 0;
+    let alone = false;
+    let waiting = 0;
+    let wakers = [];
+    const wake = () => { const w = wakers; wakers = []; for (const f of w)
+        f(); };
+    const until = async (ok) => { while (!ok())
+        await new Promise((r) => wakers.push(r)); };
+    return {
+        async shared(job) {
+            await until(() => !alone && !waiting);
+            sharing++;
+            try {
+                return await job();
+            }
+            finally {
+                sharing--;
+                wake();
+            }
+        },
+        async alone(job) {
+            waiting++;
+            await until(() => !alone && !sharing);
+            waiting--;
+            alone = true;
+            try {
+                return await job();
+            }
+            finally {
+                alone = false;
+                wake();
+            }
+        },
+    };
+}
 export async function openMemory(o) {
     const dir = resolve(o.home);
-    mkdirSync(dir, { recursive: true });
     const log = o.log ?? (() => { });
-    // Embedding: here, or on a server.
+    // Private: the logs, the index and the word vectors are everything that was added (a shared machine).
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    try {
+        chmodSync(dir, 0o700);
+    }
+    catch (err) {
+        log(`[memtrails] ${dir} could not be made private: ${err.message}`);
+    }
+    const release = lockHome(dir);
+    const started = { proc: null, local: null };
+    try {
+        return await openIn(dir, o, log, started, release);
+    }
+    catch (err) {
+        await started.proc?.stopAndWait();
+        await started.local?.close().catch(() => undefined);
+        release();
+        throw err;
+    }
+}
+async function openIn(dir, o, log, started, release) {
+    // Embedding: here, or on a server — the options' own, else the one the environment names.
     const e = o.embedding ?? {};
-    const local = !e.server || e.modelDir
+    const server = e.server ?? (e.modelDir ? null : resolveEmbedRoute());
+    const local = !server || e.modelDir
         ? await loadLocalEmbedder(e.modelDir ?? await ensureModel({ log }), { threads: e.threads ?? 2 })
         : null;
-    const embed = e.server ? remoteEmbedder({ route: () => e.server }) : local.embed;
+    started.local = local;
+    const embed = server ? remoteEmbedder({ route: () => server }) : local.embed;
     const encode = local?.words ?? null;
     const words = new WordStore(join(dir, 'words.bin'));
     await words.load();
@@ -64,6 +219,7 @@ export async function openMemory(o) {
     }
     else {
         proc = new QdrantProcess({ binary: q && 'binary' in q ? q.binary : await ensureQdrant({ log }), dir: join(dir, 'qdrant'), log });
+        started.proc = proc;
         await proc.start();
         endpoint = () => proc.current();
     }
@@ -75,24 +231,70 @@ export async function openMemory(o) {
     await ltm.load();
     const usage = new UsageBook(join(dir, 'usage.json'));
     const book = new RecallBook(join(dir, 'recall.jsonl'));
+    void book.trim(); // the documented cap: once an open, and every so many events while open (recall-log.ts)
     const ask = typeof o.llm === 'function' ? o.llm : o.llm ? openAiAsk(o.llm) : defaultAsk;
-    const base = { client, embed, stm, ltm, log, aborted: () => false, ask };
+    /** Set by close(): what runs stops at its next step. */
+    let closing = false;
+    const aborted = () => closing;
+    const base = { client, embed, stm, ltm, log, aborted, ask };
+    const rerankEnv = o.rerank === undefined ? resolveRerankRoute() : null;
     const rerank = o.rerank === false ? undefined
         : typeof o.rerank === 'function' ? o.rerank
             : o.rerank && typeof o.rerank === 'object' ? remoteReranker({ route: () => o.rerank })
-                : encode ? wordReranker(encode, words) : undefined;
+                : rerankEnv ? remoteReranker({ route: () => rerankEnv })
+                    : encode ? wordReranker(encode, words) : undefined;
     const deps = { client, embed, rerank, used: (ids, what) => usage.note(ids, what), record: (ev) => book.record(ev) };
-    // The index is derived: empty while the logs are not (a new Qdrant, a deleted folder), it is written again.
+    // The index is derived: written again from the logs when it is empty while they are not (a new Qdrant, a deleted
+    // folder), or was written in another layout — or in one not written down (an index from before the layouts were).
     const rebuild = new MemoryRebuild({
-        path: join(dir, 'rebuild.json'), stm, ltm, aborted: () => false,
+        path: join(dir, 'rebuild.json'), stm, ltm, aborted,
         writeStm: (xs) => writeStm(base, xs), writeLtm: (xs) => writeLtm(base, xs),
     });
-    if (rebuild.pending() || (!(await client.count()) && (stm.size() || ltm.size()))) {
+    const indexed = await client.count();
+    const layout = indexLayout(dir);
+    if (layout !== INDEX_LAYOUT) {
+        if (indexed) {
+            log(`[memtrails] the index was written in layout ${layout ?? 'unknown'}, not ${INDEX_LAYOUT}: writing it again from the logs`);
+            rebuild.begin(); // before the drop: a crash after it resumes
+            await client.dropCollection();
+            await client.ensureCollection();
+        }
+        saveIndexLayout(dir);
+    }
+    if (rebuild.pending() || (!indexed && (stm.size() || ltm.size()))) {
         log('[memtrails] writing the index from the logs');
         if (!rebuild.pending())
             rebuild.begin();
         await rebuild.run();
     }
+    // consolidate, sleep, rebuild, indexWords and remove one at a time: each reads the logs and writes the index from
+    // them — two consolidations pay the model twice and write stale copies over each other, a rebuild clears what
+    // another is writing, a remove during a consolidation would see its record's episode written back after it.
+    let queue = Promise.resolve();
+    const exclusive = (job) => {
+        const run = queue.then(() => {
+            if (closing)
+                throw new Error('memory is closed');
+            return job();
+        });
+        queue = run.catch(() => undefined);
+        return run;
+    };
+    let closed = null;
+    /** Every call (add, search, open, …) shares a turn; close() takes it alone, so it lets go of Qdrant, the model and the
+     *  home only once the calls in flight are done. */
+    const calls = turns();
+    /** Adds share a turn; a rebuild takes it alone. An add beside a rebuild could write its points into the index about to
+     *  be dropped and log its record after the rebuild read the log: logged, never indexed, and skipped as unchanged when
+     *  added again. */
+    const adding = turns();
+    const check = () => { if (closing)
+        throw new Error('memory is closed'); };
+    /** A call's shared turn; refused once close() began, also when it waited for its turn meanwhile. */
+    const call = async (job) => {
+        check();
+        return calls.shared(async () => { check(); return job(); });
+    };
     /** The text word by word scores an item by: its key, as search returns it. */
     const keyText = (title, summary) => `${title}\n${summary}`;
     /** A found item's whole text for the room: a record's own, or what consolidation wrote. */
@@ -110,7 +312,7 @@ export async function openMemory(o) {
         if (!encode)
             return 0;
         const todo = [...new Set(texts)].filter((t) => t.trim() && !words.get(t));
-        for (let i = 0; i < todo.length; i += BATCH) {
+        for (let i = 0; i < todo.length && !closing; i += BATCH) {
             const part = todo.slice(i, i + BATCH);
             const encoded = await encode(part, 'document');
             await words.put(part.map((t, n) => [t, packWords(encoded[n])]));
@@ -121,7 +323,8 @@ export async function openMemory(o) {
         const room = `${x.request}\n${x.reply}`.slice(0, ROOM_TEXT);
         return [keyText(stmTitle(x), rawKey(x)), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b))];
     });
-    const write = async (entries, total) => {
+    const write = (entries, total) => call(() => adding.shared(async () => {
+        check(); // it may have waited for a rebuild that a close stopped
         const fresh = entries.filter((x) => stm.meta(x.uid)?.fp !== x.fingerprint);
         for (let i = 0; i < fresh.length; i += BATCH) {
             const part = fresh.slice(i, i + BATCH);
@@ -131,6 +334,22 @@ export async function openMemory(o) {
         }
         stm.compact();
         return { added: fresh.length, unchanged: total - fresh.length };
+    }));
+    /** Unlocked: consolidate and sleep run it inside their own turn. */
+    const indexWords = async () => {
+        if (!encode)
+            return 0;
+        const texts = [...stm.all()].flatMap((x) => wordsOfEntries([x]));
+        for (const [uid, m] of ltm.entries()) {
+            if (m.status !== 'active')
+                continue;
+            const it = ltm.get(uid);
+            if (!it)
+                continue;
+            const room = `${it.title}\n${it.summary}\n${it.body}`.slice(0, ROOM_TEXT);
+            texts.push(keyText(it.title, it.summary), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b)));
+        }
+        return encodeMissing(texts);
     };
     const memory = {
         async add(records) {
@@ -172,24 +391,32 @@ export async function openMemory(o) {
             await flush();
             return total;
         },
-        async remove(ids) {
+        remove: (ids) => exclusive(async () => {
             const uids = ids.map((x) => recordUid(typeof x === 'string' ? { id: x } : x)).filter((u) => stm.meta(u));
             if (!uids.length)
                 return 0;
-            await client.deleteItems(uids.flatMap((u) => [stmItem(u), `ep:${u}`]));
+            const episodes = uids.map((u) => `ep:${u}`);
+            await client.deleteItems([...uids.map(stmItem), ...episodes]);
             stm.remove(uids);
+            // A kept episode is the record's own text: gone from the long-term log too, or a rebuild indexes it again and the
+            // profiles go on reading it.
+            ltm.remove(episodes);
+            await Promise.all([stm.flush(), ltm.flush()]);
             return uids.length;
-        },
-        search: (s) => memorySearch(deps, s),
-        searchText: async (s) => searchText(await memorySearch(deps, s), s.order ?? 'relevance'),
+        }),
+        search: (s) => call(() => memorySearch(deps, s)),
+        searchText: (s) => call(async () => searchText(await memorySearch(deps, s), s.order ?? 'relevance')),
         async room(s) {
+            check();
             if (!encode)
                 throw new Error('the room needs the local embedding model (embedding.modelDir, or no embedding.server)');
-            const found = await memorySearch(deps, { ...s, limit: s.limit ?? 20 });
-            return sentenceRoom(s.query, found.map(roomItem), encode, words, s.chars ?? ROOM_CHARS);
+            return call(async () => {
+                const found = await memorySearch(deps, { ...s, limit: s.limit ?? 20 });
+                return sentenceRoom(s.query, found.map(roomItem), encode, words, s.chars ?? ROOM_CHARS);
+            });
         },
         async open(id) {
-            const p = await memoryOpen(deps, id);
+            const p = await call(() => memoryOpen(deps, id));
             if (!p)
                 return null;
             // A record: its kind as given. Something learned: the records it was learned from, whole.
@@ -200,12 +427,12 @@ export async function openMemory(o) {
                 .map((x) => ({ id: stmItem(x.uid), when: x.ts, title: x.title, body: episodeBody(x), ...(x.record ? { record: x.record } : {}) }));
             return sources.length ? { ...p, sources } : p;
         },
-        explore: (s) => memoryExplore(deps, s),
-        gather: (s, g) => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}), ...(g?.notes ? { notes: true } : {}) }),
-        async consolidate() {
+        explore: (s) => call(() => memoryExplore(deps, s)),
+        gather: (s, g) => call(() => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}), ...(g?.notes ? { notes: true } : {}) })),
+        consolidate: () => exclusive(async () => {
             const worker = new Consolidator({ ...base, concurrency: o.concurrency ?? 4 });
             const total = { windows: 0, added: 0, merged: 0, updated: 0, superseded: 0, linked: 0, dropped: 0, episodes: 0, observed: 0, cued: 0 };
-            for (let round = 0; round < 1000 && worker.pending().length; round++) {
+            for (let round = 0; round < 1000 && !closing && worker.pending().length; round++) {
                 const r = await worker.run(64);
                 for (const k of Object.keys(total))
                     if (typeof r[k] === 'number')
@@ -217,45 +444,45 @@ export async function openMemory(o) {
                 if (!r.windows)
                     break;
             }
-            await memory.indexWords();
+            await indexWords();
             return total;
-        },
-        async sleep() {
+        }),
+        sleep: () => exclusive(async () => {
             const r = await new Sleep({ ...base, usage, statePath: join(dir, 'sleep.json') }).run();
-            await memory.indexWords();
+            await indexWords();
             return r;
-        },
-        async indexWords() {
-            if (!encode)
-                return 0;
-            const texts = [...stm.all()].flatMap((x) => wordsOfEntries([x]));
-            for (const [uid, m] of ltm.entries()) {
-                if (m.status !== 'active')
-                    continue;
-                const it = ltm.get(uid);
-                if (!it)
-                    continue;
-                const room = `${it.title}\n${it.summary}\n${it.body}`.slice(0, ROOM_TEXT);
-                texts.push(keyText(it.title, it.summary), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b)));
-            }
-            return encodeMissing(texts);
-        },
-        async rebuild() {
+        }),
+        indexWords: () => exclusive(indexWords),
+        rebuild: () => exclusive(() => adding.alone(async () => {
+            rebuild.begin(); // before the drop: a crash, or a close, after it resumes at the next open
             await client.dropCollection();
             await client.ensureCollection();
-            rebuild.begin();
             await rebuild.run();
-        },
+        })),
         tools: () => buildAgentTools({
-            search: (s) => memorySearch(deps, s),
+            search: (s) => memory.search(s),
             open: (id) => memory.open(id),
-            explore: (s) => memoryExplore(deps, s),
+            explore: (s) => memory.explore(s),
         }),
         stats: () => ({ records: stm.size(), items: [...ltm.entries()].filter(([, m]) => m.status === 'active').length, words: words.size }),
-        async close() {
-            usage.flush();
-            await proc?.stopAndWait();
-            await local?.close();
+        close() {
+            closed ??= (async () => {
+                closing = true;
+                await queue; // the job running stops at its next step; those waiting are refused
+                // The adds and searches in flight end first; those waiting for their turn, and every call after, are refused.
+                await calls.alone(async () => {
+                    try {
+                        await book.settled();
+                        usage.flush();
+                        await proc?.stopAndWait();
+                        await local?.close();
+                    }
+                    finally {
+                        release();
+                    }
+                });
+            })();
+            return closed;
         },
     };
     return memory;

package/dist/records.js

@@ -70,7 +70,7 @@ export function entryOf(r, now, turn = 0) {
  * Turns before the first request (an assistant that opened) are the first record's own.
  */
 export function conversationEntries(c, now) {
-    const start = timeOf(c.at) ?? now;
+    const began = timeOf(c.at);
     const out = [];
     let request = [];
     let reply = [];
@@ -79,10 +79,13 @@ export function conversationEntries(c, now) {
         if (!request.length && !reply.length)
             return;
         const n = out.length;
+        // A time only the caller gave is part of the fingerprint: one defaulted from when it is added would make the same
+        // conversation added again a changed one — read, embedded and consolidated again.
+        const given = at ?? (began === undefined ? undefined : began + n * 60_000);
         const e = entryOf({ id: `${c.id}#${n}`, source: c.source, collection: c.collection, thread: c.id, kind: 'conversation',
             title: request.join('\n').split('\n').find((l) => l.trim())?.trim() || reply.join('\n').split('\n').find((l) => l.trim())?.trim() || c.id,
-            text: request.join('\n'), at: at ?? start + n * 60_000 }, now, n);
-        out.push({ ...e, request: request.join('\n') || e.title, reply: reply.join('\n'), said: reply.join('\n') });
+            text: request.join('\n'), at: given }, now, n);
+        out.push({ ...e, ts: given ?? now + n * 60_000, request: request.join('\n') || e.title, reply: reply.join('\n'), said: reply.join('\n') });
         request = [];
         reply = [];
         at = undefined;

package/dist/tools/agent-tools.d.ts

@@ -33,7 +33,6 @@ export interface ToolProvider {
         limit?: number;
     }): Promise<Explored>;
 }
-type CrossMemoryHit = RecalledItem;
 /** A date as a reader reckons with it — the weekday too ("the Friday before …"): 2023-07-14 → "Fri 14 Jul 2023". */
 export declare function readableDate(d: string): string;
 /** How long something lasted, as a reader reckons with it: "since Feb 2022", "Nov 2021 – May 2022". */
@@ -41,12 +40,10 @@ export declare function lastedText(h: {
     startAt?: unknown;
     endAt?: unknown;
 }): string;
-/** All of a profile — what memory knows about one subject, one dated fact a line (core/profiles.ts) — up to this:
- *  measured, a profile cut like any other item lost most of its gain. */
-export declare const PROFILE_SHOWN = 1500;
 /** What memory_search shows: an observation as one dated line (when it happened, and when it was told when that
  *  differs), a profile whole (up to PROFILE_SHOWN), anything else as its title line and summary — shorter when many
  *  are asked for. Best first, or by time. */
-export declare function searchText(hits: CrossMemoryHit[], order?: 'relevance' | 'time'): string;
+export declare function searchText(hits: RecalledItem[], order?: 'relevance' | 'time'): string;
+/** memory_search, memory_open and memory_explore (when the provider explores) over a memory. */
 export declare function buildAgentTools(p: ToolProvider): AgentTool[];
 export {};