memtrails 0.4.0 → 0.4.2

package/dist/core/qdrant-process.js

@@ -126,6 +126,12 @@ export class QdrantProcess {
     stopping = false;
     deaths = [];
     restartTimer = null;
+    /** A restart's start while it runs: stopAndWait waits for it, and for what it began to be gone. */
+    restarting = null;
+    /** Counts stop()s: a start that sees it move while it starts gives up — before it spawns too, where `child` is null. */
+    stops = 0;
+    /** This instance's children that have not exited: stopAndWait waits for each, a restart's own included. */
+    children = new Set();
     opts;
     /** `dir` may be relative: Qdrant runs inside it, so every path it is given is made absolute first. */
     constructor(opts) {
@@ -139,11 +145,14 @@ export class QdrantProcess {
      *  caller was told, and may not be there to stop it. */
     async start() {
         this.stopping = false;
+        const stops = this.stops;
         if (this.endpoint && this.child && this.child.exitCode === null)
             return this.endpoint;
         mkdirSync(this.opts.dir, { recursive: true, mode: 0o700 });
         await this.reapLeftover();
         const port = await freePort();
+        if (this.stops !== stops)
+            throw new Error('Qdrant was stopped while it started');
         const apiKey = randomBytes(24).toString('hex');
         const config = join(this.opts.dir, 'config.yaml');
         writeFileSync(config, qdrantConfig(this.opts.dir, port), { mode: 0o600 });
@@ -160,6 +169,10 @@ export class QdrantProcess {
             closeSync(fd);
         }
         this.child = child;
+        if (child.pid) {
+            this.children.add(child);
+            child.once('exit', () => this.children.delete(child));
+        }
         // A binary that is missing or cannot run is an 'error', never an 'exit' — unheard, it is thrown in the host.
         let failed = null;
         child.on('error', (err) => { failed ??= err; });
@@ -294,16 +307,21 @@ export class QdrantProcess {
             this.restartTimer = null;
             if (this.stopping)
                 return;
-            this.start().catch((err) => {
+            // Held until its start ends: a stop() meanwhile makes that start give up and stop what it began.
+            const run = this.start().then(() => undefined, (err) => {
+                if (this.stopping)
+                    return; // given up for a stop: nothing is down
                 this.opts.onDown?.(err.message);
-                if (!this.stopping)
-                    this.restartLater(err.message);
-            });
+                this.restartLater(err.message);
+            }).finally(() => { if (this.restarting === run)
+                this.restarting = null; });
+            this.restarting = run;
         }, wait);
         this.restartTimer.unref?.();
     }
     stop() {
         this.stopping = true;
+        this.stops++;
         if (this.restartTimer) {
             clearTimeout(this.restartTimer);
             this.restartTimer = null;
@@ -314,15 +332,19 @@ export class QdrantProcess {
         if (c)
             c.kill('SIGTERM');
     }
-    /** Stop and wait for the process to be gone — before its storage is deleted. */
+    /** Stop and wait for the process to be gone — before its storage is deleted. A restart that was starting one ends
+     *  first, and the Qdrant it began is waited for too: none is left running after a close. */
     async stopAndWait(ms = 10_000) {
-        const c = this.child;
+        const restart = this.restarting;
         this.stop();
-        if (!c || c.exitCode !== null || c.signalCode !== null)
-            return;
-        await new Promise((resolve) => {
+        const within = (p) => new Promise((resolve) => {
             const t = setTimeout(resolve, ms);
-            c.once('exit', () => { clearTimeout(t); resolve(); });
+            const done = () => { clearTimeout(t); resolve(); };
+            p.then(done, done);
         });
+        if (restart)
+            await within(restart);
+        await Promise.all([...this.children].filter((c) => c.exitCode === null && c.signalCode === null)
+            .map((c) => within(new Promise((exited) => c.once('exit', exited)))));
     }
 }

package/dist/core/rebuild.js

@@ -69,8 +69,11 @@ export class MemoryRebuild {
         let skip = after !== null;
         let batch = [];
         for (const it of all) {
-            if (cancelled())
+            if (cancelled()) {
+                // Stopped (a newer begin, a close): what is being written ends before this returns, not into the next index.
+                await Promise.allSettled(queue.map((q) => q.p));
                 return false;
+            }
             if (skip) {
                 if (it.uid === after)
                     skip = false;

package/dist/core/recall.js

@@ -11,7 +11,7 @@ import { pointId } from './qdrant-client.js';
 import { queryText } from './embedder.js';
 import { explore } from './graph.js';
 import { RERANK_TOP, rerankedOrder } from './reranker.js';
-import { children, cut, querySparse } from './text.js';
+import { children, cut, flat, querySparse } from './text.js';
 import { dateSpan, LONG_SUMMARY_CHUNKS } from './writer.js';
 import { collectionKey } from '../records.js';
 /** What search may show: active items, never a throwaway run, never superseded or archived knowledge. `kinds`: a
@@ -137,7 +137,7 @@ export async function memorySearch(d, q) {
     return out;
 }
 /** For each hit whose best point was a chunk of its text, that chunk — found again from the text (writer.ts chunks a
- *  request's whole body; an item's body, and a summary too long for one key vector). */
+ *  request's whole body; an item's body, and a summary too long for one key vector) — unless the key starts with it. */
 async function passages(d, hits) {
     const chunked = hits.filter((h) => h.point && h.point !== pointId(h.item, 0));
     if (!chunked.length)
@@ -153,7 +153,9 @@ async function passages(d, hits) {
         const summary = String(p.summary ?? '');
         const kids = p.tier === 'stm' ? children(body) : [...children(body), ...(children(summary).length >= LONG_SUMMARY_CHUNKS ? children(summary) : [])];
         const k = kids.findIndex((_, i) => pointId(h.item, i + 1) === h.point);
-        if (k >= 0)
+        // A chunk that is the key's own start is not a passage found elsewhere: a short record's only chunk is its whole
+        // key, the two points tie, and either may come back first.
+        if (k >= 0 && !flat(summary).startsWith(kids[k]))
             out.set(h.item, kids[k]);
     }
     return out;

package/dist/core/sleep.js

@@ -18,6 +18,7 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { Consolidator, learnedAt, RELATED, relateBatches, SAME } from './consolidate.js';
 import { docText } from './embedder.js';
+import { allOrAbort } from '../llm.js';
 import { pointId } from './qdrant-client.js';
 import { writeProfiles } from './profiles.js';
 import { writeLtm } from './writer.js';
@@ -115,10 +116,11 @@ export class Sleep {
         const inFlight = Math.max(1, this.d.relateInFlight ?? RELATE_IN_FLIGHT);
         let next = 0;
         let unanswered = false;
-        await Promise.all(Array.from({ length: Math.min(inFlight, batches.length) }, async () => {
-            while (!unanswered && next < batches.length && !this.d.aborted()) {
+        // A lane that throws stops the others, their asks too (allOrAbort).
+        await allOrAbort(Array.from({ length: Math.min(inFlight, batches.length) }, () => async (signal) => {
+            while (!unanswered && next < batches.length && !this.d.aborted() && !signal.aborted) {
                 const b = batches[next++];
-                const got = await worker.relate(b);
+                const got = await worker.relate(b, signal);
                 if (!(got instanceof Map)) {
                     unanswered = true;
                     r.paused ??= got.paused;
@@ -154,8 +156,9 @@ export class Sleep {
             r.merged++;
         });
         let at = 0;
-        await Promise.all(Array.from({ length: Math.max(1, this.d.applyAtOnce ?? 1) }, async () => {
-            while (at < pairs.length && !this.d.aborted())
+        // An answer that fails to apply stops the other lanes at their next pair.
+        await allOrAbort(Array.from({ length: Math.max(1, this.d.applyAtOnce ?? 1) }, () => async (signal) => {
+            while (at < pairs.length && !this.d.aborted() && !signal.aborted)
                 await apply(pairs[at++]);
         }));
         if (this.d.aborted())

package/dist/core/text.d.ts

@@ -12,6 +12,8 @@ export declare function fold(text: string): string;
 /** Words for the keyword channel, in any language: folded (fold()); identifiers kept whole AND split at `_ . - /`; in
  *  scripts without spaces (Chinese, Japanese, Thai…) the segmenter's words. */
 export declare function tokens(text: string): string[];
+/** Whitespace runs as one space, the ends trimmed. */
+export declare const flat: (s: string) => string;
 /** A text's first `n` words, as written (whitespace flattened). */
 export declare function firstWords(text: string, n: number): string;
 /** 32-bit FNV-1a: a term's index in the sparse vector. */

package/dist/core/text.js

@@ -44,7 +44,8 @@ function wordSpans(text) {
             at.push([seg.index, seg.index + seg.segment.length]);
     return at;
 }
-const flat = (s) => s.replace(/\s+/g, ' ').trim();
+/** Whitespace runs as one space, the ends trimmed. */
+export const flat = (s) => s.replace(/\s+/g, ' ').trim();
 /** A text's first `n` words, as written (whitespace flattened). */
 export function firstWords(text, n) {
     const at = wordSpans(text);

package/dist/importers/files.js

@@ -8,7 +8,8 @@
  * front-matter fields go into metadata (`author` also to the record's author). A file longer than `maxChars` is cut
  * into parts with ids `path#1`, `path#2`, … sharing the thread `path`.
  */
-import { readdir, readFile, stat } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { lstat, open, readdir } from 'node:fs/promises';
 import { basename, extname, join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { timeOf } from '../records.js';
@@ -151,6 +152,24 @@ function decodeFile(bytes) {
         return null; // binary
     return decodeBytes(bytes);
 }
+/**
+ * A file opened for reading, never through a link in its last part (`O_NOFOLLOW`), and without waiting on a pipe swapped
+ * in for it (`O_NONBLOCK`, which a regular file ignores); `'link'` when it is one. Where there is no `O_NOFOLLOW`
+ * (Windows), a link is refused by looking first.
+ */
+async function openNoFollow(path) {
+    const { O_RDONLY, O_NOFOLLOW, O_NONBLOCK } = constants;
+    if (O_NOFOLLOW === undefined)
+        return (await lstat(path)).isSymbolicLink() ? 'link' : open(path, 'r');
+    try {
+        return await open(path, O_RDONLY | O_NOFOLLOW | (O_NONBLOCK ?? 0));
+    }
+    catch (err) {
+        if (err.code === 'ELOOP')
+            return 'link';
+        throw err;
+    }
+}
 /** Records from the text files in a folder and its subfolders. */
 export async function* fromFiles(dir, opts = {}) {
     const root = resolve(dir);
@@ -163,12 +182,31 @@ export async function* fromFiles(dir, opts = {}) {
         if (opts.exclude?.(path))
             continue;
         const abs = join(root, path);
-        const info = await stat(abs);
-        if (info.size > maxBytes) {
-            opts.onSkip?.(path, `larger than ${maxBytes} bytes`);
+        // Opened without following a link, and measured and read from what was opened: a note swapped for a link after the
+        // folder was listed (to a key outside it, say) is not read through it.
+        const file = await openNoFollow(abs);
+        if (file === 'link') {
+            opts.onSkip?.(path, 'not a file: it became a link after the folder was read');
             continue;
         }
-        const raw = decodeFile(await readFile(abs));
+        let info;
+        let bytes;
+        try {
+            info = await file.stat();
+            if (!info.isFile()) {
+                opts.onSkip?.(path, 'not a file: it changed after the folder was read');
+                continue;
+            }
+            if (info.size > maxBytes) {
+                opts.onSkip?.(path, `larger than ${maxBytes} bytes`);
+                continue;
+            }
+            bytes = await file.readFile();
+        }
+        finally {
+            await file.close();
+        }
+        const raw = decodeFile(bytes);
         if (raw === null) {
             opts.onSkip?.(path, 'not text');
             continue;

package/dist/importers/jsonl.d.ts

@@ -1,3 +1,9 @@
+/**
+ * JSON Lines: one JSON object per line, read a line at a time. With no `map`, each line is a record as it is
+ * (`{ "id": "…", "text": "…", "title": …, "at": … }`); with one, `map(obj)` turns each object into a record, or into
+ * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`; so is a
+ * line longer than 64 MiB, which is never held whole.
+ */
 import { type MemoryRecord } from '../records.js';
 /** Turns one parsed line into a record, or null to leave it out. */
 export type JsonlMap = (obj: unknown, lineNumber: number) => MemoryRecord | null | undefined;

package/dist/importers/jsonl.js

@@ -1,11 +1,11 @@
 /**
  * JSON Lines: one JSON object per line, read a line at a time. With no `map`, each line is a record as it is
  * (`{ "id": "…", "text": "…", "title": …, "at": … }`); with one, `map(obj)` turns each object into a record, or into
- * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`.
+ * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`; so is a
+ * line longer than 64 MiB, which is never held whole.
  */
-import { createReadStream } from 'node:fs';
-import { createInterface } from 'node:readline';
 import { invalidRecord } from '../records.js';
+import { byteLines, LineTooLong } from './lines.js';
 import { record } from './text.js';
 /** Records from a JSON Lines file; `map` may be given here or in the options. */
 export async function* fromJsonl(path, map, opts = {}) {
@@ -15,10 +15,15 @@ export async function* fromJsonl(path, map, opts = {}) {
         stats.skipped++;
         o.onSkip?.(line, reason, n);
     };
-    const lines = createInterface({ input: createReadStream(path, { encoding: 'utf8' }), crlfDelay: Infinity });
     let n = 0;
-    for await (const raw of lines) {
+    for await (const bytes of byteLines(path)) {
         n++;
+        if (bytes instanceof LineTooLong) {
+            stats.lines++;
+            skip('', bytes.reason, n);
+            continue;
+        }
+        const raw = bytes.toString('utf8');
         const line = n === 1 ? raw.replace(/^/, '') : raw;
         if (!line.trim())
             continue;

package/dist/importers/lines.d.ts

@@ -0,0 +1,13 @@
+/** The longest line read: one past it is passed over, with a reason. */
+export declare const MAX_LINE_BYTES: number;
+/** A line longer than the bound, in its place: its bytes are passed over, never held. */
+export declare class LineTooLong {
+    readonly reason: string;
+    constructor(reason: string);
+}
+/**
+ * A file's lines as `readline` gives them — ended by `\n`, `\r\n` or a lone `\r`, without their ends, and no empty line
+ * after the last end — as bytes, each decoded by the caller. One longer than `max` bytes comes as a `LineTooLong`, and
+ * the lines after it follow.
+ */
+export declare function byteLines(path: string, max?: number): AsyncGenerator<Buffer | LineTooLong>;

package/dist/importers/lines.js

@@ -0,0 +1,70 @@
+/**
+ * A file read a line at a time, as bytes, with a bound on a line: Node's `readline` has none but V8's longest string
+ * (and fails there), so one hostile line held the whole file in memory.
+ */
+import { createReadStream } from 'node:fs';
+/** The longest line read: one past it is passed over, with a reason. */
+export const MAX_LINE_BYTES = 64 * 1024 * 1024;
+/** A line longer than the bound, in its place: its bytes are passed over, never held. */
+export class LineTooLong {
+    reason;
+    constructor(reason) {
+        this.reason = reason;
+    }
+}
+/**
+ * A file's lines as `readline` gives them — ended by `\n`, `\r\n` or a lone `\r`, without their ends, and no empty line
+ * after the last end — as bytes, each decoded by the caller. One longer than `max` bytes comes as a `LineTooLong`, and
+ * the lines after it follow.
+ */
+export async function* byteLines(path, max = MAX_LINE_BYTES) {
+    let parts = [];
+    let held = 0;
+    let read = 0;
+    let long = false;
+    let afterCr = false;
+    const line = () => {
+        const out = long ? new LineTooLong(`a line longer than ${max} bytes (${read})`) : Buffer.concat(parts, held);
+        parts = [];
+        held = 0;
+        read = 0;
+        long = false;
+        return out;
+    };
+    for await (const chunk of createReadStream(path)) {
+        let at = 0;
+        while (at < chunk.length) {
+            // A `\n` right after a `\r` is the same end.
+            if (afterCr) {
+                afterCr = false;
+                if (chunk[at] === 0x0a) {
+                    at++;
+                    continue;
+                }
+            }
+            const lf = chunk.indexOf(0x0a, at);
+            const cr = chunk.indexOf(0x0d, at);
+            const end = lf < 0 ? cr : cr < 0 ? lf : Math.min(lf, cr);
+            const piece = chunk.subarray(at, end < 0 ? chunk.length : end);
+            if (!long) {
+                if (held + piece.length > max) {
+                    long = true;
+                    parts = [];
+                    held = 0;
+                }
+                else if (piece.length) {
+                    parts.push(Buffer.from(piece)); // a copy: the stream's chunk is not this line's to keep
+                    held += piece.length;
+                }
+            }
+            read += piece.length;
+            if (end < 0)
+                break;
+            afterCr = chunk[end] === 0x0d;
+            at = end + 1;
+            yield line();
+        }
+    }
+    if (read > 0)
+        yield line();
+}

package/dist/importers/mbox.js

@@ -11,11 +11,10 @@
  * metadata { from, to, cc, inReplyTo }.
  */
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { basename, extname } from 'node:path';
-import { createInterface } from 'node:readline';
 import { timeOf } from '../records.js';
+import { byteLines, LineTooLong } from './lines.js';
 import { decodeBytes, htmlToText, record } from './text.js';
 function entityOf(lines) {
     const headers = new Map();
@@ -307,8 +306,6 @@ function messageRecord(lines, fromLine, source, opts) {
 export async function* fromMbox(path, opts = {}) {
     const source = opts.source ?? basename(path, extname(path));
     const max = opts.maxMessageBytes ?? 32 * 1024 * 1024;
-    // Latin-1 keeps every byte as one character: each part is decoded later in its own charset.
-    const lines = createInterface({ input: createReadStream(path, { encoding: 'latin1' }), crlfDelay: Infinity });
     let msg = null;
     let fromLine = '';
     let size = 0;
@@ -323,7 +320,15 @@ export async function* fromMbox(path, opts = {}) {
         index++;
         return r;
     };
-    for await (const line of lines) {
+    for await (const bytes of byteLines(path)) {
+        // A line past the bound (64 MiB): the message is past its size from here, as a long one is.
+        if (bytes instanceof LineTooLong) {
+            size = max + 1;
+            previousBlank = false;
+            continue;
+        }
+        // Latin-1 keeps every byte as one character: each part is decoded later in its own charset.
+        const line = bytes.toString('latin1');
         // A new message: a `From ` line at the start or after an empty line, carrying a time (`From - Mon Jan 1 00:00:00 2024`).
         if (line.startsWith('From ') && previousBlank && (msg === null || /\d:\d\d/.test(line))) {
             const r = finish();

package/dist/llm.d.ts

@@ -12,6 +12,9 @@ export interface LlmRequest {
     timeoutMs?: number;
     /** What a failure cost, said after its reason ("so no summary was written"). */
     consequence?: string;
+    /** Aborted when no one will read the answer any more (an ask beside it failed, the run stopped): an ask that honours
+     *  it stops paying for one. */
+    signal?: AbortSignal;
 }
 export type LlmAnswer = {
     answer: Record<string, unknown>;
@@ -48,6 +51,12 @@ export interface OpenAiAskOptions {
     retries?: number;
     fetchImpl?: typeof fetch;
 }
+/**
+ * `Promise.all` for asks side by side, each handed a signal: the first that throws aborts the others — an ask that
+ * honours `signal` stops, and none is paid for an answer no one reads — and they have all settled when it is thrown.
+ * `outer` (the caller's) aborts them too.
+ */
+export declare function allOrAbort<T>(jobs: Array<(signal: AbortSignal) => Promise<T>>, outer?: AbortSignal): Promise<T[]>;
 export declare function openAiAsk(o?: OpenAiAskOptions): Ask;
 /** The model the environment names (MEMTRAILS_MODEL and a key), asked when no `ask` is given. */
 export declare const defaultAsk: Ask;

package/dist/llm.js

@@ -8,6 +8,40 @@
 import { redactString } from './core/redact.js';
 const OPENAI = 'https://api.openai.com/v1';
 const why = (reason, req) => `${reason}${req.consequence ? `, ${req.consequence}` : ''}`;
+/** A wait that an abort ends early. */
+const pause = (ms, signal) => new Promise((ok) => {
+    if (signal?.aborted)
+        return ok();
+    const done = () => { clearTimeout(t); signal?.removeEventListener('abort', done); ok(); };
+    const t = setTimeout(done, ms);
+    signal?.addEventListener('abort', done, { once: true });
+});
+/**
+ * `Promise.all` for asks side by side, each handed a signal: the first that throws aborts the others — an ask that
+ * honours `signal` stops, and none is paid for an answer no one reads — and they have all settled when it is thrown.
+ * `outer` (the caller's) aborts them too.
+ */
+export async function allOrAbort(jobs, outer) {
+    const c = new AbortController();
+    const follow = () => c.abort(outer?.reason);
+    if (outer?.aborted)
+        follow();
+    else
+        outer?.addEventListener('abort', follow, { once: true });
+    // Each started in order, at once, as Promise.all over async functions would.
+    const runs = jobs.map(async (job) => job(c.signal));
+    try {
+        return await Promise.all(runs);
+    }
+    catch (err) {
+        c.abort(err);
+        await Promise.allSettled(runs);
+        throw err;
+    }
+    finally {
+        outer?.removeEventListener('abort', follow);
+    }
+}
 export function openAiAsk(o = {}) {
     return async (req) => {
         const model = req.model ?? o.model ?? process.env.MEMTRAILS_MODEL?.trim();
@@ -24,18 +58,24 @@ export function openAiAsk(o = {}) {
             text: { format: { type: 'json_schema', name: req.name ?? 'memory', strict: true, schema: req.schema } },
         });
         const tries = Math.max(1, (o.retries ?? 5) + 1);
+        const stopped = { unavailable: why('the ask was stopped', req) };
         let last = '';
         for (let attempt = 0; attempt < tries; attempt++) {
             if (attempt)
-                await new Promise((ok) => setTimeout(ok, Math.min(60_000, 1000 * 2 ** (attempt - 1)) * (0.5 + Math.random())));
+                await pause(Math.min(60_000, 1000 * 2 ** (attempt - 1)) * (0.5 + Math.random()), req.signal);
+            if (req.signal?.aborted)
+                return stopped;
+            const timeout = AbortSignal.timeout(req.timeoutMs ?? 180_000);
             let res;
             try {
                 res = await (o.fetchImpl ?? fetch)(`${base}/responses`, {
-                    method: 'POST', body, signal: AbortSignal.timeout(req.timeoutMs ?? 180_000),
+                    method: 'POST', body, signal: req.signal ? AbortSignal.any([req.signal, timeout]) : timeout,
                     headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
                 });
             }
             catch (err) {
+                if (req.signal?.aborted)
+                    return stopped;
                 last = err.message || err.name;
                 continue;
             }

package/dist/memory.d.ts

@@ -106,7 +106,8 @@ export interface Memory {
         words: number;
     };
     /** Stops a consolidate, sleep, rebuild or indexWords at its next step (a rebuild resumes at the next open) and waits
-     *  for it, then releases Qdrant, the model and the home. */
+     *  for it and for the adds and searches in flight, then releases Qdrant, the model and the home. Every call after it
+     *  is refused ("memory is closed"). */
     close(): Promise<void>;
 }
 export declare function openMemory(o: MemoryOptions): Promise<Memory>;