memtrails 0.3.0 → 0.4.1

package/dist/importers/files.js

@@ -8,7 +8,8 @@
  * front-matter fields go into metadata (`author` also to the record's author). A file longer than `maxChars` is cut
  * into parts with ids `path#1`, `path#2`, … sharing the thread `path`.
  */
-import { readdir, readFile, stat } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { lstat, open, readdir } from 'node:fs/promises';
 import { basename, extname, join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { timeOf } from '../records.js';
@@ -31,6 +32,19 @@ function scalar(raw) {
 function listValue(items) {
     return items.map((i) => scalar(i)).filter((i) => i !== null && i !== '').join(', ');
 }
+/**
+ * One front-matter line as key and value: TOML `key = value`, YAML `key: value` (a `:` then a space or the end). Split
+ * at the first `=` or `:` by hand: `^([^=]*?)\s*=` tried a run of spaces at every position, quadratic on a long line.
+ */
+function keyValue(line, toml) {
+    const at = line.indexOf(toml ? '=' : ':');
+    if (at <= 0 || (toml ? /[\s=#[]/ : /[\s:#-]/).test(line[0]))
+        return null;
+    const value = line.slice(at + 1);
+    if (!toml && value && !/^\s/.test(value))
+        return null;
+    return [line.slice(0, at).trimEnd(), value];
+}
 /** YAML (`---`) or TOML (`+++`) front matter, flat fields only: scalars, lists (joined with `, `) and block text. */
 export function frontMatter(text) {
     const m = /^(---|\+\+\+)[ \t]*\r?\n([\s\S]*?)\r?\n(?:---|\+\+\+|\.\.\.)[ \t]*(?:\r?\n|$)/.exec(text);
@@ -40,11 +54,11 @@ export function frontMatter(text) {
     const fields = {};
     const lines = m[2].split(/\r?\n/);
     for (let i = 0; i < lines.length; i++) {
-        const kv = toml ? /^([^\s=#[][^=]*?)\s*=\s*(.*)$/.exec(lines[i]) : /^([^\s:#-][^:]*?)\s*:(?:\s+(.*)|\s*)$/.exec(lines[i]);
+        const kv = keyValue(lines[i], toml);
         if (!kv)
             continue;
-        const key = kv[1].trim().replace(/^["']|["']$/g, '');
-        const value = (kv[2] ?? '').trim();
+        const key = kv[0].trim().replace(/^["']|["']$/g, '');
+        const value = kv[1].trim();
         if (value.startsWith('[') && value.endsWith(']')) {
             fields[key] = listValue(value.slice(1, -1).split(','));
         }
@@ -64,24 +78,54 @@ export function frontMatter(text) {
                 fields[key] = listValue(items);
         }
         else {
-            fields[key] = scalar(value.replace(/\s+#.*$/, ''));
+            const comment = value.search(/\s#/); // not /\s+#.*$/: quadratic on a long run of spaces
+            fields[key] = scalar(comment < 0 ? value : value.slice(0, comment));
         }
     }
     return { fields, body: text.slice(m[0].length) };
 }
+/** Title rules see at most this much of a file, a line at a time: per-line regexes over a whole hostile file were slow. */
+const TITLE_SCAN = 64 * 1024;
+/** An RST underline (or overline): one punctuation character, three or more times, then spaces at most. */
+function rstRule(line) {
+    const t = line?.trimEnd() ?? ''; // not /[ \t]+$/: quadratic on a long run of spaces
+    return t.length >= 3 && '=-~^"\'`#*+'.includes(t[0]) && t === t[0].repeat(t.length);
+}
 function titleOf(ext, body, raw) {
     if (ext === '.html' || ext === '.htm' || ext === '.xhtml')
         return htmlTitle(raw);
-    if (ext === '.org')
-        return /^#\+title:[ \t]*(.+)$/im.exec(body)?.[1].trim() || /^\*+[ \t]+(.+)$/m.exec(body)?.[1].trim();
+    const lines = body.slice(0, TITLE_SCAN).split(/\r\n?|\n/);
+    const first = (f) => {
+        for (let i = 0; i < lines.length; i++) {
+            const t = f(lines[i], i);
+            if (t !== undefined)
+                return t;
+        }
+        return undefined;
+    };
+    if (ext === '.org') {
+        return first((l) => (/^#\+title:/i.test(l) && l.length > 8 ? l.slice(8).trim() : undefined)) ||
+            first((l) => (/^\*+[ \t]/.test(l) && l.replace(/^\*+/, '').trim() ? l.replace(/^\*+/, '').trim() : undefined));
+    }
     if (ext === '.rst')
-        return /^(?:([=\-~^"'`#*+])\1+\r?\n)?[ \t]*(\S[^\n]*?)[ \t]*\r?\n([=\-~^"'`#*+])\3{2,}[ \t]*$/m.exec(body)?.[2];
+        return first((l, i) => (l.trim() && rstRule(lines[i + 1]) ? l.trim() : undefined));
     if (ext === '.adoc' || ext === '.asciidoc')
-        return /^=[ \t]+(.+)$/m.exec(body)?.[1].trim();
-    const atx = /^#[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/m.exec(body)?.[1].trim();
+        return first((l) => (/^=[ \t]/.test(l) && l.slice(1).trim() ? l.slice(1).trim() : undefined));
+    // `# Title`, a closing run of `#` after a space dropped (by hand: `[ \t]+#+$` is quadratic on a run of spaces).
+    const atx = first((l) => {
+        if (!/^#[ \t]/.test(l))
+            return undefined;
+        const t = l.slice(1).trim();
+        let j = t.length;
+        while (j > 0 && t[j - 1] === '#')
+            j--;
+        const title = j > 0 && j < t.length && /[ \t]/.test(t[j - 1]) ? t.slice(0, j).trimEnd() : t;
+        return title || undefined;
+    });
     if (atx)
         return atx;
-    return /^(\S[^\n]*?)[ \t]*\r?\n=+[ \t]*$/m.exec(body)?.[1]; // a Setext `Title\n=====`
+    // A Setext `Title\n=====`.
+    return first((l, i) => (/^\S/.test(l) && /^=+[ \t]*$/.test(lines[i + 1] ?? '') ? l.trimEnd() : undefined));
 }
 async function* walk(root, rel, o) {
     const entries = await readdir(join(root, rel), { withFileTypes: true });
@@ -108,6 +152,24 @@ function decodeFile(bytes) {
         return null; // binary
     return decodeBytes(bytes);
 }
+/**
+ * A file opened for reading, never through a link in its last part (`O_NOFOLLOW`), and without waiting on a pipe swapped
+ * in for it (`O_NONBLOCK`, which a regular file ignores); `'link'` when it is one. Where there is no `O_NOFOLLOW`
+ * (Windows), a link is refused by looking first.
+ */
+async function openNoFollow(path) {
+    const { O_RDONLY, O_NOFOLLOW, O_NONBLOCK } = constants;
+    if (O_NOFOLLOW === undefined)
+        return (await lstat(path)).isSymbolicLink() ? 'link' : open(path, 'r');
+    try {
+        return await open(path, O_RDONLY | O_NOFOLLOW | (O_NONBLOCK ?? 0));
+    }
+    catch (err) {
+        if (err.code === 'ELOOP')
+            return 'link';
+        throw err;
+    }
+}
 /** Records from the text files in a folder and its subfolders. */
 export async function* fromFiles(dir, opts = {}) {
     const root = resolve(dir);
@@ -120,12 +182,31 @@ export async function* fromFiles(dir, opts = {}) {
         if (opts.exclude?.(path))
             continue;
         const abs = join(root, path);
-        const info = await stat(abs);
-        if (info.size > maxBytes) {
-            opts.onSkip?.(path, `larger than ${maxBytes} bytes`);
+        // Opened without following a link, and measured and read from what was opened: a note swapped for a link after the
+        // folder was listed (to a key outside it, say) is not read through it.
+        const file = await openNoFollow(abs);
+        if (file === 'link') {
+            opts.onSkip?.(path, 'not a file: it became a link after the folder was read');
             continue;
         }
-        const raw = decodeFile(await readFile(abs));
+        let info;
+        let bytes;
+        try {
+            info = await file.stat();
+            if (!info.isFile()) {
+                opts.onSkip?.(path, 'not a file: it changed after the folder was read');
+                continue;
+            }
+            if (info.size > maxBytes) {
+                opts.onSkip?.(path, `larger than ${maxBytes} bytes`);
+                continue;
+            }
+            bytes = await file.readFile();
+        }
+        finally {
+            await file.close();
+        }
+        const raw = decodeFile(bytes);
         if (raw === null) {
             opts.onSkip?.(path, 'not text');
             continue;

package/dist/importers/git.js

@@ -60,7 +60,9 @@ export async function* fromGit(repoDir, opts = {}) {
     }
     args.push('--', ...(opts.paths ?? []));
     const maxFiles = opts.maxFiles ?? 50;
-    const child = spawn(opts.git ?? 'git', args, { cwd: repoDir, stdio: ['ignore', 'pipe', 'pipe'] });
+    // `-C`, not `cwd`: on Windows a command is looked up in its working folder first, so `cwd: repoDir` ran a `git.exe`
+    // planted in the repository being imported.
+    const child = spawn(opts.git ?? 'git', ['-C', resolve(repoDir), ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
     let stderr = '';
     child.stderr.setEncoding('utf8');
     child.stderr.on('data', (d) => {

package/dist/importers/jsonl.d.ts

@@ -1,3 +1,9 @@
+/**
+ * JSON Lines: one JSON object per line, read a line at a time. With no `map`, each line is a record as it is
+ * (`{ "id": "…", "text": "…", "title": …, "at": … }`); with one, `map(obj)` turns each object into a record, or into
+ * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`; so is a
+ * line longer than 64 MiB, which is never held whole.
+ */
 import { type MemoryRecord } from '../records.js';
 /** Turns one parsed line into a record, or null to leave it out. */
 export type JsonlMap = (obj: unknown, lineNumber: number) => MemoryRecord | null | undefined;

package/dist/importers/jsonl.js

@@ -1,11 +1,11 @@
 /**
  * JSON Lines: one JSON object per line, read a line at a time. With no `map`, each line is a record as it is
  * (`{ "id": "…", "text": "…", "title": …, "at": … }`); with one, `map(obj)` turns each object into a record, or into
- * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`.
+ * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`; so is a
+ * line longer than 64 MiB, which is never held whole.
  */
-import { createReadStream } from 'node:fs';
-import { createInterface } from 'node:readline';
 import { invalidRecord } from '../records.js';
+import { byteLines, LineTooLong } from './lines.js';
 import { record } from './text.js';
 /** Records from a JSON Lines file; `map` may be given here or in the options. */
 export async function* fromJsonl(path, map, opts = {}) {
@@ -15,10 +15,15 @@ export async function* fromJsonl(path, map, opts = {}) {
         stats.skipped++;
         o.onSkip?.(line, reason, n);
     };
-    const lines = createInterface({ input: createReadStream(path, { encoding: 'utf8' }), crlfDelay: Infinity });
     let n = 0;
-    for await (const raw of lines) {
+    for await (const bytes of byteLines(path)) {
         n++;
+        if (bytes instanceof LineTooLong) {
+            stats.lines++;
+            skip('', bytes.reason, n);
+            continue;
+        }
+        const raw = bytes.toString('utf8');
         const line = n === 1 ? raw.replace(/^/, '') : raw;
         if (!line.trim())
             continue;

package/dist/importers/lines.d.ts

@@ -0,0 +1,13 @@
+/** The longest line read: one past it is passed over, with a reason. */
+export declare const MAX_LINE_BYTES: number;
+/** A line longer than the bound, in its place: its bytes are passed over, never held. */
+export declare class LineTooLong {
+    readonly reason: string;
+    constructor(reason: string);
+}
+/**
+ * A file's lines as `readline` gives them — ended by `\n`, `\r\n` or a lone `\r`, without their ends, and no empty line
+ * after the last end — as bytes, each decoded by the caller. One longer than `max` bytes comes as a `LineTooLong`, and
+ * the lines after it follow.
+ */
+export declare function byteLines(path: string, max?: number): AsyncGenerator<Buffer | LineTooLong>;

package/dist/importers/lines.js

@@ -0,0 +1,70 @@
+/**
+ * A file read a line at a time, as bytes, with a bound on a line: Node's `readline` has none but V8's longest string
+ * (and fails there), so one hostile line held the whole file in memory.
+ */
+import { createReadStream } from 'node:fs';
+/** The longest line read: one past it is passed over, with a reason. */
+export const MAX_LINE_BYTES = 64 * 1024 * 1024;
+/** A line longer than the bound, in its place: its bytes are passed over, never held. */
+export class LineTooLong {
+    reason;
+    constructor(reason) {
+        this.reason = reason;
+    }
+}
+/**
+ * A file's lines as `readline` gives them — ended by `\n`, `\r\n` or a lone `\r`, without their ends, and no empty line
+ * after the last end — as bytes, each decoded by the caller. One longer than `max` bytes comes as a `LineTooLong`, and
+ * the lines after it follow.
+ */
+export async function* byteLines(path, max = MAX_LINE_BYTES) {
+    let parts = [];
+    let held = 0;
+    let read = 0;
+    let long = false;
+    let afterCr = false;
+    const line = () => {
+        const out = long ? new LineTooLong(`a line longer than ${max} bytes (${read})`) : Buffer.concat(parts, held);
+        parts = [];
+        held = 0;
+        read = 0;
+        long = false;
+        return out;
+    };
+    for await (const chunk of createReadStream(path)) {
+        let at = 0;
+        while (at < chunk.length) {
+            // A `\n` right after a `\r` is the same end.
+            if (afterCr) {
+                afterCr = false;
+                if (chunk[at] === 0x0a) {
+                    at++;
+                    continue;
+                }
+            }
+            const lf = chunk.indexOf(0x0a, at);
+            const cr = chunk.indexOf(0x0d, at);
+            const end = lf < 0 ? cr : cr < 0 ? lf : Math.min(lf, cr);
+            const piece = chunk.subarray(at, end < 0 ? chunk.length : end);
+            if (!long) {
+                if (held + piece.length > max) {
+                    long = true;
+                    parts = [];
+                    held = 0;
+                }
+                else if (piece.length) {
+                    parts.push(Buffer.from(piece)); // a copy: the stream's chunk is not this line's to keep
+                    held += piece.length;
+                }
+            }
+            read += piece.length;
+            if (end < 0)
+                break;
+            afterCr = chunk[end] === 0x0d;
+            at = end + 1;
+            yield line();
+        }
+    }
+    if (read > 0)
+        yield line();
+}

package/dist/importers/mbox.js

@@ -11,11 +11,10 @@
  * metadata { from, to, cc, inReplyTo }.
  */
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { basename, extname } from 'node:path';
-import { createInterface } from 'node:readline';
 import { timeOf } from '../records.js';
+import { byteLines, LineTooLong } from './lines.js';
 import { decodeBytes, htmlToText, record } from './text.js';
 function entityOf(lines) {
     const headers = new Map();
@@ -148,7 +147,10 @@ function parseParams(raw) {
 function quotedPrintable(lines) {
     let s = '';
     for (const [i, raw] of lines.entries()) {
-        const line = raw.replace(/[ \t\r]+$/, '');
+        let end = raw.length;
+        while (end > 0 && (raw[end - 1] === ' ' || raw[end - 1] === '\t' || raw[end - 1] === '\r'))
+            end--; // /[ \t\r]+$/ is quadratic on a run of spaces
+        const line = raw.slice(0, end);
         if (line.endsWith('='))
             s += line.slice(0, -1);
         else
@@ -257,12 +259,24 @@ function ids(raw) {
     const bracketed = [...v.matchAll(/<([^<>\s]+)>/g)].map((m) => m[1]);
     return bracketed.length ? bracketed : v.split(/\s+/).filter((x) => x.includes('@'));
 }
+/** `Name <address>` → Name, `address (Name)` → Name. By hand: `^\s*(\S+@\S+)\s*\(` is quadratic on `a@a@a@…`. */
 function personOf(from) {
-    const named = /^(.*?)\s*<([^<>]*)>\s*$/.exec(from);
-    if (named)
-        return named[1].replace(/^"(.*)"$/, '$1').trim() || named[2].trim();
-    const commented = /^\s*(\S+@\S+)\s*\((.+)\)\s*$/.exec(from);
-    return commented ? commented[2].trim() : from.trim();
+    const t = from.trim();
+    const lt = t.lastIndexOf('<');
+    if (t.endsWith('>') && lt >= 0 && !t.slice(lt + 1, -1).includes('>')) {
+        return t.slice(0, lt).trim().replace(/^"(.*)"$/, '$1').trim() || t.slice(lt + 1, -1).trim();
+    }
+    const open = t.indexOf('(');
+    const address = open > 0 ? t.slice(0, open).trim() : '';
+    if (t.endsWith(')') && t.length - open > 2 && /^\S+$/.test(address) && address.indexOf('@') > 0 && !address.endsWith('@')) {
+        return t.slice(open + 1, -1).trim();
+    }
+    return t;
+}
+/** `(comments)` taken out of a header, up to its last `)`: past it, each `(` would scan to the end and fail. */
+function dropComments(v) {
+    const end = v.lastIndexOf(')') + 1;
+    return v.slice(0, end).replace(/\([^)]*\)/g, ' ') + v.slice(end);
 }
 /** One message's lines (raw bytes as Latin-1 characters, no line endings) as a record, or null when it says nothing. */
 function messageRecord(lines, fromLine, source, opts) {
@@ -277,7 +291,7 @@ function messageRecord(lines, fromLine, source, opts) {
     if (!text && !subject)
         return null;
     const from = h('from');
-    const date = h('date').replace(/\([^)]*\)/g, ' ').trim();
+    const date = dropComments(h('date')).trim();
     const at = timeOf(date) ?? timeOf(fromLine.replace(/^From \S+\s+/, '').trim());
     const own = ids(e.headers.get('message-id'))[0];
     const id = own ?? createHash('sha256').update([from, date, subject, text.slice(0, 4000)].join('\u0000')).digest('hex').slice(0, 32);
@@ -292,8 +306,6 @@ function messageRecord(lines, fromLine, source, opts) {
 export async function* fromMbox(path, opts = {}) {
     const source = opts.source ?? basename(path, extname(path));
     const max = opts.maxMessageBytes ?? 32 * 1024 * 1024;
-    // Latin-1 keeps every byte as one character: each part is decoded later in its own charset.
-    const lines = createInterface({ input: createReadStream(path, { encoding: 'latin1' }), crlfDelay: Infinity });
     let msg = null;
     let fromLine = '';
     let size = 0;
@@ -308,7 +320,15 @@ export async function* fromMbox(path, opts = {}) {
         index++;
         return r;
     };
-    for await (const line of lines) {
+    for await (const bytes of byteLines(path)) {
+        // A line past the bound (64 MiB): the message is past its size from here, as a long one is.
+        if (bytes instanceof LineTooLong) {
+            size = max + 1;
+            previousBlank = false;
+            continue;
+        }
+        // Latin-1 keeps every byte as one character: each part is decoded later in its own charset.
+        const line = bytes.toString('latin1');
         // A new message: a `From ` line at the start or after an empty line, carrying a time (`From - Mon Jan 1 00:00:00 2024`).
         if (line.startsWith('From ') && previousBlank && (msg === null || /\d:\d\d/.test(line))) {
             const r = finish();

package/dist/importers/slack.js

@@ -9,7 +9,7 @@
  * channel's name. kind `message`. Joins, leaves, topic changes and the like are left out.
  */
 import { readdir, readFile, stat } from 'node:fs/promises';
-import { basename, join, resolve } from 'node:path';
+import { basename, dirname, join, resolve } from 'node:path';
 import { record } from './text.js';
 /** Message subtypes left out by default: events about the channel, not things said in it. */
 export const SLACK_SKIPPED_SUBTYPES = ['channel_join', 'channel_leave', 'channel_topic', 'channel_purpose', 'channel_name',
@@ -89,7 +89,10 @@ export async function* fromSlackExport(dir, opts = {}) {
     for (const c of conversations) {
         if (wanted && !wanted.has(c.name) && !wanted.has(c.id))
             continue;
-        const folder = join(root, c.folder);
+        // The folder's name comes from the export's own lists: `../x`, `a/b` or an absolute path must not lead out of it.
+        const folder = resolve(root, c.folder);
+        if (!c.folder || dirname(folder) !== root)
+            continue;
         if (!(await stat(folder).catch(() => undefined))?.isDirectory())
             continue;
         const days = (await readdir(folder)).filter((f) => /^\d{4}-\d{2}-\d{2}\.json$/.test(f)).sort();

package/dist/importers/text.js

@@ -53,13 +53,23 @@ export function decodeEntities(s) {
 }
 const BLOCK = 'address|article|aside|blockquote|caption|center|dd|details|dialog|div|dl|dt|fieldset|figcaption|figure|footer|form|' +
     'h[1-6]|header|hr|main|nav|ol|p|pre|section|summary|table|tbody|thead|tfoot|tr|ul';
+/**
+ * `f` applied to `s` up to its last `>`, the rest kept. Every tag pattern here ends in `[^>]*>`: past the last `>` each
+ * `<` is a start that scans to the end and fails, so a run of unclosed `<a<a<a…` was quadratic; before it, each scan
+ * stops at the next `>` and matches.
+ */
+function beforeLastGt(s, f) {
+    const end = s.lastIndexOf('>') + 1;
+    return end === s.length ? f(s) : f(s.slice(0, end)) + s.slice(end);
+}
 /** Removes every `<tag …>…</tag>` (nested ones too), for `blockquote`: the quoted earlier message in an HTML reply. */
 function dropElements(html, tag) {
     const re = new RegExp(`<(/?)${tag}\\b[^>]*>`, 'gi');
+    const head = html.slice(0, html.lastIndexOf('>') + 1); // see beforeLastGt
     let out = '';
     let depth = 0;
     let from = 0;
-    for (let m = re.exec(html); m; m = re.exec(html)) {
+    for (let m = re.exec(head); m; m = re.exec(head)) {
         if (!m[1]) {
             if (depth === 0)
                 out += html.slice(from, m.index);
@@ -73,11 +83,76 @@ function dropElements(html, tag) {
     }
     return depth === 0 ? out + html.slice(from) : out;
 }
+/**
+ * Each `<tag …>…</tag>` found by `open` (the tag's name in group 1) replaced by `f(name, element)`, scanning once:
+ * `<(script|…)\b[\s\S]*?<\/\1>` rescanned to the end for every unclosed opening tag, quadratic on `<script<script…`.
+ * An unclosed one has `element` null; `f` then gives what replaces it and everything after it, or null to leave it as
+ * written. A tag with no closing tag after it has none after a later opening tag either, so it is looked for once.
+ * The text between elements goes through `outside`.
+ */
+function replaceElements(s, open, f, outside = (t) => t) {
+    const closes = new Map();
+    const unclosed = new Set();
+    let out = '';
+    let from = 0;
+    open.lastIndex = 0;
+    for (let m = open.exec(s); m; m = open.exec(s)) {
+        const name = m[1].toLowerCase();
+        let end = -1;
+        if (!unclosed.has(name)) {
+            let close = closes.get(name);
+            if (!close)
+                closes.set(name, (close = new RegExp(`</${name}\\s*>`, 'gi')));
+            close.lastIndex = m.index + m[0].length;
+            const c = close.exec(s);
+            if (c)
+                end = c.index + c[0].length;
+            else
+                unclosed.add(name);
+        }
+        const replaced = f(name, end < 0 ? null : s.slice(m.index, end));
+        if (replaced === null)
+            continue; // left as written: look on after the opening tag
+        out += outside(s.slice(from, m.index)) + replaced;
+        if (end < 0)
+            return out;
+        from = open.lastIndex = end;
+    }
+    return out + outside(s.slice(from));
+}
+/** `<![CDATA[…]]>` as its text; an unclosed one, and everything after it, left as written. */
+function unwrapCdata(s) {
+    let out = '';
+    let from = 0;
+    for (let i = s.indexOf('<![CDATA[', from); i >= 0; i = s.indexOf('<![CDATA[', from)) {
+        const j = s.indexOf(']]>', i + 9);
+        if (j < 0)
+            break;
+        out += s.slice(from, i) + s.slice(i + 9, j);
+        from = j + 3;
+    }
+    return out + s.slice(from);
+}
+const HIDDEN = /<(script|style|head|noscript|template|svg|math|iframe|object)\b/gi;
+const PRE = /<(pre)\b/gi;
+/** Title regexes see at most this much of a page: a title is near the top, and a hostile page is not read to its end. */
+const TITLE_SCAN = 64 * 1024;
 /** The `<title>` of an HTML page, else its first `<h1>`, as text; undefined when it has neither. */
 export function htmlTitle(html) {
-    for (const re of [/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i, /<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/i]) {
-        const m = re.exec(html);
-        const t = m ? decodeEntities(m[1].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim() : '';
+    const top = html.slice(0, TITLE_SCAN);
+    const head = top.slice(0, top.lastIndexOf('>') + 1); // see beforeLastGt
+    for (const tag of ['title', 'h1']) {
+        const open = new RegExp(`<${tag}\\b[^>]*>`, 'i').exec(head);
+        if (!open)
+            continue;
+        const bodyStart = open.index + open[0].length;
+        const close = new RegExp(`</${tag}\\s*>`, 'gi');
+        close.lastIndex = bodyStart;
+        const end = close.exec(head);
+        if (!end)
+            continue;
+        const inner = beforeLastGt(head.slice(bodyStart, end.index), (h) => h.replace(/<[^>]*>/g, ' '));
+        const t = decodeEntities(inner).replace(/\s+/g, ' ').trim();
         if (t)
             return t;
     }
@@ -88,24 +163,20 @@ export function htmlTitle(html) {
  * as `- `, tags removed, entities decoded, `<pre>` kept as written. `dropQuotes` also drops `<blockquote>`s.
  */
 export function htmlToText(html, opts = {}) {
-    let s = html
-        .replace(/<!--[\s\S]*?(-->|$)/g, ' ')
-        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
-        .replace(/<(script|style|head|noscript|template|svg|math|iframe|object)\b[\s\S]*?<\/\1\s*>/gi, ' ')
-        .replace(/<(script|style)\b[\s\S]*$/i, ' '); // one left open runs to the end
+    let s = unwrapCdata(html.replace(/<!--[\s\S]*?(-->|$)/g, ' '));
+    // Hidden elements dropped; a script or style left open runs to the end, any other unclosed one is left to the tag rules.
+    s = replaceElements(s, HIDDEN, (name, element) => (element !== null || name === 'script' || name === 'style' ? ' ' : null));
     if (opts.dropQuotes)
         s = dropElements(s, 'blockquote');
     // Whitespace in the source is one space, except inside <pre>.
-    s = s.split(/(<pre\b[\s\S]*?<\/pre\s*>)/i)
-        .map((piece, i) => (i % 2 ? piece.replace(/\r\n?/g, '\n') : piece.replace(/\s+/g, ' ')))
-        .join('');
-    s = s
+    s = replaceElements(s, PRE, (_name, element) => element?.replace(/\r\n?/g, '\n') ?? null, (t) => t.replace(/\s+/g, ' '));
+    s = beforeLastGt(s, (h) => h
         .replace(/<br\b[^>]*>/gi, '\n')
         .replace(/<li\b[^>]*>/gi, '\n- ')
         .replace(new RegExp(`</?(?:${BLOCK})\\b[^>]*>`, 'gi'), '\n')
         .replace(/<\/t[dh]\s*>/gi, '\t')
         .replace(/<\/?[a-zA-Z][^>]*>/g, '')
-        .replace(/<[!?][^>]*>/g, '');
+        .replace(/<[!?][^>]*>/g, ''));
     s = decodeEntities(s);
     return tidy(s);
 }

package/dist/llm.d.ts

@@ -1,10 +1,3 @@
-/**
- * The language model memory asks, for consolidation, sleep and gathering — never for adding or searching. One structured
- * ask: a system text, a prompt and a JSON Schema the answer must follow (strict structured output).
- *
- * `openAiAsk` speaks the OpenAI Responses API, so any provider or local server that serves it works (`baseUrl`). Bring
- * any other model by passing your own `Ask`. Every call's tokens can be counted (`onUsage`).
- */
 export interface LlmRequest {
     /** What this is for, in a word (`memory`). */
     purpose: string;
@@ -19,6 +12,9 @@ export interface LlmRequest {
     timeoutMs?: number;
     /** What a failure cost, said after its reason ("so no summary was written"). */
     consequence?: string;
+    /** Aborted when no one will read the answer any more (an ask beside it failed, the run stopped): an ask that honours
+     *  it stops paying for one. */
+    signal?: AbortSignal;
 }
 export type LlmAnswer = {
     answer: Record<string, unknown>;
@@ -41,7 +37,7 @@ export interface LlmUsage {
     outputTokens: number;
 }
 export interface OpenAiAskOptions {
-    /** Default: $MEMTRAILS_LLM_API_KEY, else $OPENAI_API_KEY. */
+    /** Default: $MEMTRAILS_LLM_API_KEY, else $OPENAI_API_KEY — the latter only for OpenAI's own address. */
     apiKey?: string;
     /** Default: $MEMTRAILS_MODEL. */
     model?: string;
@@ -55,6 +51,12 @@ export interface OpenAiAskOptions {
     retries?: number;
     fetchImpl?: typeof fetch;
 }
+/**
+ * `Promise.all` for asks side by side, each handed a signal: the first that throws aborts the others — an ask that
+ * honours `signal` stops, and none is paid for an answer no one reads — and they have all settled when it is thrown.
+ * `outer` (the caller's) aborts them too.
+ */
+export declare function allOrAbort<T>(jobs: Array<(signal: AbortSignal) => Promise<T>>, outer?: AbortSignal): Promise<T[]>;
 export declare function openAiAsk(o?: OpenAiAskOptions): Ask;
 /** The model the environment names (MEMTRAILS_MODEL and a key), asked when no `ask` is given. */
 export declare const defaultAsk: Ask;