memtrails 0.4.0 → 0.4.1

package/dist/memory.js

@@ -136,6 +136,44 @@ function saveIndexLayout(dir) {
     writeFileSync(`${path}.tmp`, JSON.stringify({ layout: INDEX_LAYOUT }), { mode: 0o600 });
     renameSync(`${path}.tmp`, path);
 }
+/** Turns shared, and a turn alone: many share at once; one alone waits until no one shares, and keeps new sharers out
+ *  while it waits, so a stream of them never holds it off. */
+function turns() {
+    let sharing = 0;
+    let alone = false;
+    let waiting = 0;
+    let wakers = [];
+    const wake = () => { const w = wakers; wakers = []; for (const f of w)
+        f(); };
+    const until = async (ok) => { while (!ok())
+        await new Promise((r) => wakers.push(r)); };
+    return {
+        async shared(job) {
+            await until(() => !alone && !waiting);
+            sharing++;
+            try {
+                return await job();
+            }
+            finally {
+                sharing--;
+                wake();
+            }
+        },
+        async alone(job) {
+            waiting++;
+            await until(() => !alone && !sharing);
+            waiting--;
+            alone = true;
+            try {
+                return await job();
+            }
+            finally {
+                alone = false;
+                wake();
+            }
+        },
+    };
+}
 export async function openMemory(o) {
     const dir = resolve(o.home);
     const log = o.log ?? (() => { });
@@ -243,6 +281,20 @@ async function openIn(dir, o, log, started, release) {
         return run;
     };
     let closed = null;
+    /** Every call (add, search, open, …) shares a turn; close() takes it alone, so it lets go of Qdrant, the model and the
+     *  home only once the calls in flight are done. */
+    const calls = turns();
+    /** Adds share a turn; a rebuild takes it alone. An add beside a rebuild could write its points into the index about to
+     *  be dropped and log its record after the rebuild read the log: logged, never indexed, and skipped as unchanged when
+     *  added again. */
+    const adding = turns();
+    const check = () => { if (closing)
+        throw new Error('memory is closed'); };
+    /** A call's shared turn; refused once close() began, also when it waited for its turn meanwhile. */
+    const call = async (job) => {
+        check();
+        return calls.shared(async () => { check(); return job(); });
+    };
     /** The text word by word scores an item by: its key, as search returns it. */
     const keyText = (title, summary) => `${title}\n${summary}`;
     /** A found item's whole text for the room: a record's own, or what consolidation wrote. */
@@ -271,7 +323,8 @@ async function openIn(dir, o, log, started, release) {
         const room = `${x.request}\n${x.reply}`.slice(0, ROOM_TEXT);
         return [keyText(stmTitle(x), rawKey(x)), ...sentenceSpans(room).map(([a, b]) => room.slice(a, b))];
     });
-    const write = async (entries, total) => {
+    const write = (entries, total) => call(() => adding.shared(async () => {
+        check(); // it may have waited for a rebuild that a close stopped
         const fresh = entries.filter((x) => stm.meta(x.uid)?.fp !== x.fingerprint);
         for (let i = 0; i < fresh.length; i += BATCH) {
             const part = fresh.slice(i, i + BATCH);
@@ -281,7 +334,7 @@ async function openIn(dir, o, log, started, release) {
         }
         stm.compact();
         return { added: fresh.length, unchanged: total - fresh.length };
-    };
+    }));
     /** Unlocked: consolidate and sleep run it inside their own turn. */
     const indexWords = async () => {
         if (!encode)
@@ -351,16 +404,19 @@ async function openIn(dir, o, log, started, release) {
             await Promise.all([stm.flush(), ltm.flush()]);
             return uids.length;
         }),
-        search: (s) => memorySearch(deps, s),
-        searchText: async (s) => searchText(await memorySearch(deps, s), s.order ?? 'relevance'),
+        search: (s) => call(() => memorySearch(deps, s)),
+        searchText: (s) => call(async () => searchText(await memorySearch(deps, s), s.order ?? 'relevance')),
         async room(s) {
+            check();
             if (!encode)
                 throw new Error('the room needs the local embedding model (embedding.modelDir, or no embedding.server)');
-            const found = await memorySearch(deps, { ...s, limit: s.limit ?? 20 });
-            return sentenceRoom(s.query, found.map(roomItem), encode, words, s.chars ?? ROOM_CHARS);
+            return call(async () => {
+                const found = await memorySearch(deps, { ...s, limit: s.limit ?? 20 });
+                return sentenceRoom(s.query, found.map(roomItem), encode, words, s.chars ?? ROOM_CHARS);
+            });
         },
         async open(id) {
-            const p = await memoryOpen(deps, id);
+            const p = await call(() => memoryOpen(deps, id));
             if (!p)
                 return null;
             // A record: its kind as given. Something learned: the records it was learned from, whole.
@@ -371,8 +427,8 @@ async function openIn(dir, o, log, started, release) {
                 .map((x) => ({ id: stmItem(x.uid), when: x.ts, title: x.title, body: episodeBody(x), ...(x.record ? { record: x.record } : {}) }));
             return sources.length ? { ...p, sources } : p;
         },
-        explore: (s) => memoryExplore(deps, s),
-        gather: (s, g) => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}), ...(g?.notes ? { notes: true } : {}) }),
+        explore: (s) => call(() => memoryExplore(deps, s)),
+        gather: (s, g) => call(() => memoryGather(deps, s, { ask, ...(s.limit ? { limit: s.limit } : {}), ...(g?.notes ? { notes: true } : {}) })),
         consolidate: () => exclusive(async () => {
             const worker = new Consolidator({ ...base, concurrency: o.concurrency ?? 4 });
             const total = { windows: 0, added: 0, merged: 0, updated: 0, superseded: 0, linked: 0, dropped: 0, episodes: 0, observed: 0, cued: 0 };
@@ -397,31 +453,34 @@ async function openIn(dir, o, log, started, release) {
             return r;
         }),
         indexWords: () => exclusive(indexWords),
-        rebuild: () => exclusive(async () => {
+        rebuild: () => exclusive(() => adding.alone(async () => {
             rebuild.begin(); // before the drop: a crash, or a close, after it resumes at the next open
             await client.dropCollection();
             await client.ensureCollection();
             await rebuild.run();
-        }),
+        })),
         tools: () => buildAgentTools({
-            search: (s) => memorySearch(deps, s),
+            search: (s) => memory.search(s),
             open: (id) => memory.open(id),
-            explore: (s) => memoryExplore(deps, s),
+            explore: (s) => memory.explore(s),
         }),
         stats: () => ({ records: stm.size(), items: [...ltm.entries()].filter(([, m]) => m.status === 'active').length, words: words.size }),
         close() {
             closed ??= (async () => {
                 closing = true;
                 await queue; // the job running stops at its next step; those waiting are refused
-                try {
-                    await book.settled();
-                    usage.flush();
-                    await proc?.stopAndWait();
-                    await local?.close();
-                }
-                finally {
-                    release();
-                }
+                // The adds and searches in flight end first; those waiting for their turn, and every call after, are refused.
+                await calls.alone(async () => {
+                    try {
+                        await book.settled();
+                        usage.flush();
+                        await proc?.stopAndWait();
+                        await local?.close();
+                    }
+                    finally {
+                        release();
+                    }
+                });
             })();
             return closed;
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memtrails",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Retrieval and exploration for any text — e-mail, chat, notes, documents, news, commits, conversations. Searchable at once, on your own machine, in any language; optional consolidation into typed knowledge with a language model.",
   "keywords": [
     "retrieval",