memtrails 0.2.0 → 0.4.0

package/dist/core/writer.d.ts

@@ -18,7 +18,8 @@ export declare function writeStm(d: WriterDeps, entries: StmEntry[]): Promise<vo
  *  either there are, so the two never meet. */
 export declare const cuePoint: (k: number) => number;
 /** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
- *  `eventAt`, or a date an agent searches by) — null for anything else. */
+ *  `eventAt`, or a date an agent searches by) — null for anything else, a month or day that does not exist too
+ *  (2024-02-30 is not 1 March). */
 export declare function dateSpan(d: string | undefined): {
     from: number;
     to: number;

package/dist/core/writer.js

@@ -36,11 +36,12 @@ export async function writeStm(d, entries) {
     const points = [];
     let at = 0;
     for (const { e, body, kids } of plan) {
-        const common = { item: stmItem(e.uid), tier: 'stm', status: 'active', kind: 'episode', scope: 'project', project: e.project, session: e.session, ts: e.ts, synthetic: e.synthetic };
+        // The record's kind (`engine`) on every point: a kinds filter must find a record by any of its chunks.
+        const common = { item: stmItem(e.uid), tier: 'stm', status: 'active', kind: 'episode', engine: e.engine, scope: 'project', project: e.project, session: e.session, ts: e.ts, synthetic: e.synthetic };
         points.push({
             id: pointId(stmItem(e.uid), 0),
             vector: { dense: Array.from(vecs[at]), bm25: docSparse(`${stmTitle(e)}\n${body}`) },
-            payload: { ...common, part: 'key', engine: e.engine, cwd: e.cwd ?? '', where: e.title, title: stmTitle(e), summary: rawKey(e), body, files: e.files, importance: 0, ...(e.record ? { record: e.record } : {}) },
+            payload: { ...common, part: 'key', cwd: e.cwd ?? '', where: e.title, title: stmTitle(e), summary: rawKey(e), body, files: e.files, importance: 0, ...(e.record ? { record: e.record } : {}) },
         });
         kids.forEach((_, k) => points.push({ id: pointId(stmItem(e.uid), k + 1), vector: { dense: Array.from(vecs[at + 1 + k]) }, payload: { ...common, part: 'child' } }));
         at += 1 + kids.length;
@@ -53,12 +54,17 @@ export async function writeStm(d, entries) {
  *  either there are, so the two never meet. */
 export const cuePoint = (k) => -(k + 1);
 /** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
- *  `eventAt`, or a date an agent searches by) — null for anything else. */
+ *  `eventAt`, or a date an agent searches by) — null for anything else, a month or day that does not exist too
+ *  (2024-02-30 is not 1 March). */
 export function dateSpan(d) {
     const m = /^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$/.exec(d ?? '');
     if (!m)
         return null;
     const [y, mo, dd] = [Number(m[1]), m[2] ? Number(m[2]) - 1 : null, m[3] ? Number(m[3]) : null];
+    if (mo !== null && (mo < 0 || mo > 11))
+        return null;
+    if (mo !== null && dd !== null && (dd < 1 || new Date(Date.UTC(y, mo, dd)).getUTCMonth() !== mo))
+        return null;
     if (mo === null)
         return { from: Date.UTC(y, 0, 1), to: Date.UTC(y + 1, 0, 1) - 1 };
     if (dd === null)

package/dist/importers/files.js

@@ -31,6 +31,19 @@ function scalar(raw) {
 function listValue(items) {
     return items.map((i) => scalar(i)).filter((i) => i !== null && i !== '').join(', ');
 }
+/**
+ * One front-matter line as key and value: TOML `key = value`, YAML `key: value` (a `:` then a space or the end). Split
+ * at the first `=` or `:` by hand: `^([^=]*?)\s*=` tried a run of spaces at every position, quadratic on a long line.
+ */
+function keyValue(line, toml) {
+    const at = line.indexOf(toml ? '=' : ':');
+    if (at <= 0 || (toml ? /[\s=#[]/ : /[\s:#-]/).test(line[0]))
+        return null;
+    const value = line.slice(at + 1);
+    if (!toml && value && !/^\s/.test(value))
+        return null;
+    return [line.slice(0, at).trimEnd(), value];
+}
 /** YAML (`---`) or TOML (`+++`) front matter, flat fields only: scalars, lists (joined with `, `) and block text. */
 export function frontMatter(text) {
     const m = /^(---|\+\+\+)[ \t]*\r?\n([\s\S]*?)\r?\n(?:---|\+\+\+|\.\.\.)[ \t]*(?:\r?\n|$)/.exec(text);
@@ -40,11 +53,11 @@ export function frontMatter(text) {
     const fields = {};
     const lines = m[2].split(/\r?\n/);
     for (let i = 0; i < lines.length; i++) {
-        const kv = toml ? /^([^\s=#[][^=]*?)\s*=\s*(.*)$/.exec(lines[i]) : /^([^\s:#-][^:]*?)\s*:(?:\s+(.*)|\s*)$/.exec(lines[i]);
+        const kv = keyValue(lines[i], toml);
         if (!kv)
             continue;
-        const key = kv[1].trim().replace(/^["']|["']$/g, '');
-        const value = (kv[2] ?? '').trim();
+        const key = kv[0].trim().replace(/^["']|["']$/g, '');
+        const value = kv[1].trim();
         if (value.startsWith('[') && value.endsWith(']')) {
             fields[key] = listValue(value.slice(1, -1).split(','));
         }
@@ -64,24 +77,54 @@ export function frontMatter(text) {
                 fields[key] = listValue(items);
         }
         else {
-            fields[key] = scalar(value.replace(/\s+#.*$/, ''));
+            const comment = value.search(/\s#/); // not /\s+#.*$/: quadratic on a long run of spaces
+            fields[key] = scalar(comment < 0 ? value : value.slice(0, comment));
         }
     }
     return { fields, body: text.slice(m[0].length) };
 }
+/** Title rules see at most this much of a file, a line at a time: per-line regexes over a whole hostile file were slow. */
+const TITLE_SCAN = 64 * 1024;
+/** An RST underline (or overline): one punctuation character, three or more times, then spaces at most. */
+function rstRule(line) {
+    const t = line?.trimEnd() ?? ''; // not /[ \t]+$/: quadratic on a long run of spaces
+    return t.length >= 3 && '=-~^"\'`#*+'.includes(t[0]) && t === t[0].repeat(t.length);
+}
 function titleOf(ext, body, raw) {
     if (ext === '.html' || ext === '.htm' || ext === '.xhtml')
         return htmlTitle(raw);
-    if (ext === '.org')
-        return /^#\+title:[ \t]*(.+)$/im.exec(body)?.[1].trim() || /^\*+[ \t]+(.+)$/m.exec(body)?.[1].trim();
+    const lines = body.slice(0, TITLE_SCAN).split(/\r\n?|\n/);
+    const first = (f) => {
+        for (let i = 0; i < lines.length; i++) {
+            const t = f(lines[i], i);
+            if (t !== undefined)
+                return t;
+        }
+        return undefined;
+    };
+    if (ext === '.org') {
+        return first((l) => (/^#\+title:/i.test(l) && l.length > 8 ? l.slice(8).trim() : undefined)) ||
+            first((l) => (/^\*+[ \t]/.test(l) && l.replace(/^\*+/, '').trim() ? l.replace(/^\*+/, '').trim() : undefined));
+    }
     if (ext === '.rst')
-        return /^(?:([=\-~^"'`#*+])\1+\r?\n)?[ \t]*(\S[^\n]*?)[ \t]*\r?\n([=\-~^"'`#*+])\3{2,}[ \t]*$/m.exec(body)?.[2];
+        return first((l, i) => (l.trim() && rstRule(lines[i + 1]) ? l.trim() : undefined));
     if (ext === '.adoc' || ext === '.asciidoc')
-        return /^=[ \t]+(.+)$/m.exec(body)?.[1].trim();
-    const atx = /^#[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/m.exec(body)?.[1].trim();
+        return first((l) => (/^=[ \t]/.test(l) && l.slice(1).trim() ? l.slice(1).trim() : undefined));
+    // `# Title`, a closing run of `#` after a space dropped (by hand: `[ \t]+#+$` is quadratic on a run of spaces).
+    const atx = first((l) => {
+        if (!/^#[ \t]/.test(l))
+            return undefined;
+        const t = l.slice(1).trim();
+        let j = t.length;
+        while (j > 0 && t[j - 1] === '#')
+            j--;
+        const title = j > 0 && j < t.length && /[ \t]/.test(t[j - 1]) ? t.slice(0, j).trimEnd() : t;
+        return title || undefined;
+    });
     if (atx)
         return atx;
-    return /^(\S[^\n]*?)[ \t]*\r?\n=+[ \t]*$/m.exec(body)?.[1]; // a Setext `Title\n=====`
+    // A Setext `Title\n=====`.
+    return first((l, i) => (/^\S/.test(l) && /^=+[ \t]*$/.test(lines[i + 1] ?? '') ? l.trimEnd() : undefined));
 }
 async function* walk(root, rel, o) {
     const entries = await readdir(join(root, rel), { withFileTypes: true });

package/dist/importers/git.js

@@ -60,7 +60,9 @@ export async function* fromGit(repoDir, opts = {}) {
     }
     args.push('--', ...(opts.paths ?? []));
     const maxFiles = opts.maxFiles ?? 50;
-    const child = spawn(opts.git ?? 'git', args, { cwd: repoDir, stdio: ['ignore', 'pipe', 'pipe'] });
+    // `-C`, not `cwd`: on Windows a command is looked up in its working folder first, so `cwd: repoDir` ran a `git.exe`
+    // planted in the repository being imported.
+    const child = spawn(opts.git ?? 'git', ['-C', resolve(repoDir), ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
     let stderr = '';
     child.stderr.setEncoding('utf8');
     child.stderr.on('data', (d) => {

package/dist/importers/mbox.js

@@ -148,7 +148,10 @@ function parseParams(raw) {
 function quotedPrintable(lines) {
     let s = '';
     for (const [i, raw] of lines.entries()) {
-        const line = raw.replace(/[ \t\r]+$/, '');
+        let end = raw.length;
+        while (end > 0 && (raw[end - 1] === ' ' || raw[end - 1] === '\t' || raw[end - 1] === '\r'))
+            end--; // /[ \t\r]+$/ is quadratic on a run of spaces
+        const line = raw.slice(0, end);
         if (line.endsWith('='))
             s += line.slice(0, -1);
         else
@@ -257,12 +260,24 @@ function ids(raw) {
     const bracketed = [...v.matchAll(/<([^<>\s]+)>/g)].map((m) => m[1]);
     return bracketed.length ? bracketed : v.split(/\s+/).filter((x) => x.includes('@'));
 }
+/** `Name <address>` → Name, `address (Name)` → Name. By hand: `^\s*(\S+@\S+)\s*\(` is quadratic on `a@a@a@…`. */
 function personOf(from) {
-    const named = /^(.*?)\s*<([^<>]*)>\s*$/.exec(from);
-    if (named)
-        return named[1].replace(/^"(.*)"$/, '$1').trim() || named[2].trim();
-    const commented = /^\s*(\S+@\S+)\s*\((.+)\)\s*$/.exec(from);
-    return commented ? commented[2].trim() : from.trim();
+    const t = from.trim();
+    const lt = t.lastIndexOf('<');
+    if (t.endsWith('>') && lt >= 0 && !t.slice(lt + 1, -1).includes('>')) {
+        return t.slice(0, lt).trim().replace(/^"(.*)"$/, '$1').trim() || t.slice(lt + 1, -1).trim();
+    }
+    const open = t.indexOf('(');
+    const address = open > 0 ? t.slice(0, open).trim() : '';
+    if (t.endsWith(')') && t.length - open > 2 && /^\S+$/.test(address) && address.indexOf('@') > 0 && !address.endsWith('@')) {
+        return t.slice(open + 1, -1).trim();
+    }
+    return t;
+}
+/** `(comments)` taken out of a header, up to its last `)`: past it, each `(` would scan to the end and fail. */
+function dropComments(v) {
+    const end = v.lastIndexOf(')') + 1;
+    return v.slice(0, end).replace(/\([^)]*\)/g, ' ') + v.slice(end);
 }
 /** One message's lines (raw bytes as Latin-1 characters, no line endings) as a record, or null when it says nothing. */
 function messageRecord(lines, fromLine, source, opts) {
@@ -277,7 +292,7 @@ function messageRecord(lines, fromLine, source, opts) {
     if (!text && !subject)
         return null;
     const from = h('from');
-    const date = h('date').replace(/\([^)]*\)/g, ' ').trim();
+    const date = dropComments(h('date')).trim();
     const at = timeOf(date) ?? timeOf(fromLine.replace(/^From \S+\s+/, '').trim());
     const own = ids(e.headers.get('message-id'))[0];
     const id = own ?? createHash('sha256').update([from, date, subject, text.slice(0, 4000)].join('\u0000')).digest('hex').slice(0, 32);

package/dist/importers/slack.js

@@ -9,7 +9,7 @@
  * channel's name. kind `message`. Joins, leaves, topic changes and the like are left out.
  */
 import { readdir, readFile, stat } from 'node:fs/promises';
-import { basename, join, resolve } from 'node:path';
+import { basename, dirname, join, resolve } from 'node:path';
 import { record } from './text.js';
 /** Message subtypes left out by default: events about the channel, not things said in it. */
 export const SLACK_SKIPPED_SUBTYPES = ['channel_join', 'channel_leave', 'channel_topic', 'channel_purpose', 'channel_name',
@@ -89,7 +89,10 @@ export async function* fromSlackExport(dir, opts = {}) {
     for (const c of conversations) {
         if (wanted && !wanted.has(c.name) && !wanted.has(c.id))
             continue;
-        const folder = join(root, c.folder);
+        // The folder's name comes from the export's own lists: `../x`, `a/b` or an absolute path must not lead out of it.
+        const folder = resolve(root, c.folder);
+        if (!c.folder || dirname(folder) !== root)
+            continue;
         if (!(await stat(folder).catch(() => undefined))?.isDirectory())
             continue;
         const days = (await readdir(folder)).filter((f) => /^\d{4}-\d{2}-\d{2}\.json$/.test(f)).sort();

package/dist/importers/text.js

@@ -53,13 +53,23 @@ export function decodeEntities(s) {
 }
 const BLOCK = 'address|article|aside|blockquote|caption|center|dd|details|dialog|div|dl|dt|fieldset|figcaption|figure|footer|form|' +
     'h[1-6]|header|hr|main|nav|ol|p|pre|section|summary|table|tbody|thead|tfoot|tr|ul';
+/**
+ * `f` applied to `s` up to its last `>`, the rest kept. Every tag pattern here ends in `[^>]*>`: past the last `>` each
+ * `<` is a start that scans to the end and fails, so a run of unclosed `<a<a<a…` was quadratic; before it, each scan
+ * stops at the next `>` and matches.
+ */
+function beforeLastGt(s, f) {
+    const end = s.lastIndexOf('>') + 1;
+    return end === s.length ? f(s) : f(s.slice(0, end)) + s.slice(end);
+}
 /** Removes every `<tag …>…</tag>` (nested ones too), for `blockquote`: the quoted earlier message in an HTML reply. */
 function dropElements(html, tag) {
     const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, 'gi');
+    const head = html.slice(0, html.lastIndexOf('>') + 1); // see beforeLastGt
     let out = '';
     let depth = 0;
     let from = 0;
-    for (let m = re.exec(html); m; m = re.exec(html)) {
+    for (let m = re.exec(head); m; m = re.exec(head)) {
         if (!m[1]) {
             if (depth === 0)
                 out += html.slice(from, m.index);
@@ -73,11 +83,76 @@ function dropElements(html, tag) {
     }
     return depth === 0 ? out + html.slice(from) : out;
 }
+/**
+ * Each `<tag …>…</tag>` found by `open` (the tag's name in group 1) replaced by `f(name, element)`, scanning once:
+ * `<(script|…)\b[\s\S]*?<\/\1>` rescanned to the end for every unclosed opening tag, quadratic on `<script<script…`.
+ * An unclosed one has `element` null; `f` then gives what replaces it and everything after it, or null to leave it as
+ * written. A tag with no closing tag after it has none after a later opening tag either, so it is looked for once.
+ * The text between elements goes through `outside`.
+ */
+function replaceElements(s, open, f, outside = (t) => t) {
+    const closes = new Map();
+    const unclosed = new Set();
+    let out = '';
+    let from = 0;
+    open.lastIndex = 0;
+    for (let m = open.exec(s); m; m = open.exec(s)) {
+        const name = m[1].toLowerCase();
+        let end = -1;
+        if (!unclosed.has(name)) {
+            let close = closes.get(name);
+            if (!close)
+                closes.set(name, (close = new RegExp(`</${name}\\s*>`, 'gi')));
+            close.lastIndex = m.index + m[0].length;
+            const c = close.exec(s);
+            if (c)
+                end = c.index + c[0].length;
+            else
+                unclosed.add(name);
+        }
+        const replaced = f(name, end < 0 ? null : s.slice(m.index, end));
+        if (replaced === null)
+            continue; // left as written: look on after the opening tag
+        out += outside(s.slice(from, m.index)) + replaced;
+        if (end < 0)
+            return out;
+        from = open.lastIndex = end;
+    }
+    return out + outside(s.slice(from));
+}
+/** `<![CDATA[…]]>` as its text; an unclosed one, and everything after it, left as written. */
+function unwrapCdata(s) {
+    let out = '';
+    let from = 0;
+    for (let i = s.indexOf('<![CDATA[', from); i >= 0; i = s.indexOf('<![CDATA[', from)) {
+        const j = s.indexOf(']]>', i + 9);
+        if (j < 0)
+            break;
+        out += s.slice(from, i) + s.slice(i + 9, j);
+        from = j + 3;
+    }
+    return out + s.slice(from);
+}
+const HIDDEN = /<(script|style|head|noscript|template|svg|math|iframe|object)\b/gi;
+const PRE = /<(pre)\b/gi;
+/** Title regexes see at most this much of a page: a title is near the top, and a hostile page is not read to its end. */
+const TITLE_SCAN = 64 * 1024;
 /** The `<title>` of an HTML page, else its first `<h1>`, as text; undefined when it has neither. */
 export function htmlTitle(html) {
-    for (const re of [/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i, /<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/i]) {
-        const m = re.exec(html);
-        const t = m ? decodeEntities(m[1].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim() : '';
+    const top = html.slice(0, TITLE_SCAN);
+    const head = top.slice(0, top.lastIndexOf('>') + 1); // see beforeLastGt
+    for (const tag of ['title', 'h1']) {
+        const open = new RegExp(`<${tag}\\b[^>]*>`, 'i').exec(head);
+        if (!open)
+            continue;
+        const bodyStart = open.index + open[0].length;
+        const close = new RegExp(`</${tag}\\s*>`, 'gi');
+        close.lastIndex = bodyStart;
+        const end = close.exec(head);
+        if (!end)
+            continue;
+        const inner = beforeLastGt(head.slice(bodyStart, end.index), (h) => h.replace(/<[^>]*>/g, ' '));
+        const t = decodeEntities(inner).replace(/\s+/g, ' ').trim();
         if (t)
             return t;
     }
@@ -88,24 +163,20 @@ export function htmlTitle(html) {
  * as `- `, tags removed, entities decoded, `<pre>` kept as written. `dropQuotes` also drops `<blockquote>`s.
  */
 export function htmlToText(html, opts = {}) {
-    let s = html
-        .replace(/<!--[\s\S]*?(-->|$)/g, ' ')
-        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
-        .replace(/<(script|style|head|noscript|template|svg|math|iframe|object)\b[\s\S]*?<\/\1\s*>/gi, ' ')
-        .replace(/<(script|style)\b[\s\S]*$/i, ' '); // one left open runs to the end
+    let s = unwrapCdata(html.replace(/<!--[\s\S]*?(-->|$)/g, ' '));
+    // Hidden elements dropped; a script or style left open runs to the end, any other unclosed one is left to the tag rules.
+    s = replaceElements(s, HIDDEN, (name, element) => (element !== null || name === 'script' || name === 'style' ? ' ' : null));
     if (opts.dropQuotes)
         s = dropElements(s, 'blockquote');
     // Whitespace in the source is one space, except inside <pre>.
-    s = s.split(/(<pre\b[\s\S]*?<\/pre\s*>)/i)
-        .map((piece, i) => (i % 2 ? piece.replace(/\r\n?/g, '\n') : piece.replace(/\s+/g, ' ')))
-        .join('');
-    s = s
+    s = replaceElements(s, PRE, (_name, element) => element?.replace(/\r\n?/g, '\n') ?? null, (t) => t.replace(/\s+/g, ' '));
+    s = beforeLastGt(s, (h) => h
         .replace(/<br\b[^>]*>/gi, '\n')
         .replace(/<li\b[^>]*>/gi, '\n- ')
         .replace(new RegExp(`</?(?:${BLOCK})\\b[^>]*>`, 'gi'), '\n')
         .replace(/<\/t[dh]\s*>/gi, '\t')
         .replace(/<\/?[a-zA-Z][^>]*>/g, '')
-        .replace(/<[!?][^>]*>/g, '');
+        .replace(/<[!?][^>]*>/g, ''));
     s = decodeEntities(s);
     return tidy(s);
 }

package/dist/llm.d.ts

@@ -1,10 +1,3 @@
-/**
- * The language model memory asks, for consolidation, sleep and gathering — never for adding or searching. One structured
- * ask: a system text, a prompt and a JSON Schema the answer must follow (strict structured output).
- *
- * `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
- * any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
- */
 export interface LlmRequest {
     /** What this is for, in a word (`memory`). */
     purpose: string;
@@ -41,7 +34,7 @@ export interface LlmUsage {
     outputTokens: number;
 }
 export interface OpenAiAskOptions {
-    /** Default: $MEMTRAILS_LLM_API_KEY, else $OPENAI_API_KEY. */
+    /** Default: $MEMTRAILS_LLM_API_KEY, else $OPENAI_API_KEY — the latter only for OpenAI's own address. */
     apiKey?: string;
     /** Default: $MEMTRAILS_MODEL. */
     model?: string;

package/dist/llm.js

@@ -5,16 +5,19 @@
  * `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
  * any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
  */
+import { redactString } from './core/redact.js';
+const OPENAI = 'https://api.openai.com/v1';
 const why = (reason, req) => `${reason}${req.consequence ? `, ${req.consequence}` : ''}`;
 export function openAiAsk(o = {}) {
     return async (req) => {
         const model = req.model ?? o.model ?? process.env.MEMTRAILS_MODEL?.trim();
-        const key = o.apiKey ?? (process.env.MEMTRAILS_LLM_API_KEY?.trim() || process.env.OPENAI_API_KEY?.trim());
+        const base = (o.baseUrl ?? (process.env.MEMTRAILS_LLM_BASE_URL?.trim() || OPENAI)).replace(/\/+$/, '');
+        // An OpenAI key goes to OpenAI only: with another address set, it would hand the key to whoever serves that.
+        const key = o.apiKey ?? (process.env.MEMTRAILS_LLM_API_KEY?.trim() || (base === OPENAI ? process.env.OPENAI_API_KEY?.trim() : undefined));
         if (!model)
             return { unavailable: why('no model is set (the model option, or MEMTRAILS_MODEL)', req) };
         if (!key)
-            return { unavailable: why('no API key is set (the apiKey option, MEMTRAILS_LLM_API_KEY or OPENAI_API_KEY)', req) };
-        const base = (o.baseUrl ?? (process.env.MEMTRAILS_LLM_BASE_URL?.trim() || 'https://api.openai.com/v1')).replace(/\/+$/, '');
+            return { unavailable: why(`no API key is set (the apiKey option or MEMTRAILS_LLM_API_KEY${base === OPENAI ? ', or OPENAI_API_KEY' : ''})`, req) };
         const body = JSON.stringify({
             model, instructions: req.system, input: req.prompt, max_output_tokens: req.maxTokens ?? 12_000,
             ...(o.reasoningEffort ? { reasoning: { effort: o.reasoningEffort } } : {}),
@@ -40,8 +43,11 @@ export function openAiAsk(o = {}) {
                 last = `HTTP ${res.status}`;
                 continue;
             }
-            if (!res.ok)
-                return { unavailable: why(`the model answered HTTP ${res.status}`, req) };
+            if (!res.ok) {
+                // What the provider said (a wrong model name, a schema it refuses) — not only its status.
+                const said = redactString((await res.text().catch(() => '')).replace(/\s+/g, ' ').trim()).slice(0, 300);
+                return { unavailable: why(`the model answered HTTP ${res.status}${said ? `: ${said}` : ''}`, req) };
+            }
             try {
                 const r = (await res.json());
                 o.onUsage?.({ model: r.model ?? model, name: req.name, inputTokens: r.usage?.input_tokens ?? 0,

package/dist/memory.d.ts

@@ -14,14 +14,16 @@ export interface MemoryOptions {
      *  - default: on this machine, the model fetched on first use into the cache (218 MB);
      *  - `{ modelDir }`: on this machine, from a directory you provide;
      *  - `{ server }`: a server that serves it over the OpenAI embeddings API (text-embeddings-inference, Ollama, …).
-     *    Word by word then needs `{ modelDir }` too, or it is off. */
+     *    Word by word then needs `{ modelDir }` too, or it is off.
+     *  With neither `server` nor `modelDir`, MEMTRAILS_EMBED_URL (+ _KEY, _MODEL) names a server when it is set. */
     embedding?: {
         modelDir?: string;
         server?: EmbedRoute;
         threads?: number;
     };
     /** How the best 30 of a search are reordered: `'words'` (default with a local model: word by word, no server), a
-     *  cross-encoder server (`RerankRoute`), your own function, or `false` for the search's own order. */
+     *  cross-encoder server (`RerankRoute`), your own function, or `false` for the search's own order. Not given:
+     *  MEMTRAILS_RERANK_URL (+ _KEY, _MODEL) names a server when it is set. */
     rerank?: 'words' | RerankRoute | RecallDeps['rerank'] | false;
     /** The language model for consolidate, sleep and gather: your own `Ask`, or options for the OpenAI-compatible one.
      *  Default: the one the environment names (MEMTRAILS_MODEL and a key). */
@@ -84,7 +86,9 @@ export interface Memory {
         limit?: number;
     }): Promise<Explored>;
     /** A set for a question that counts, lists or totals: a wide search, then the language model keeps every instance. */
-    gather(q: SearchQuery): Promise<Gathered>;
+    gather(q: SearchQuery, o?: {
+        notes?: boolean;
+    }): Promise<Gathered>;
     /** Read what was added since last time and keep typed knowledge, dated observations and kept episodes. Needs a
      *  language model. Returns how many windows were read. */
     consolidate(): Promise<ConsolidationReport>;
@@ -101,6 +105,8 @@ export interface Memory {
         items: number;
         words: number;
     };
+    /** Stops a consolidate, sleep, rebuild or indexWords at its next step (a rebuild resumes at the next open) and waits
+     *  for it, then releases Qdrant, the model and the home. */
     close(): Promise<void>;
 }
 export declare function openMemory(o: MemoryOptions): Promise<Memory>;