memtrails 0.0.0-stage → 0.2.0

package/dist/core/writer.d.ts

@@ -0,0 +1,36 @@
+import { type QdrantClient } from './qdrant-client.js';
+import type { MemoryItem, StmEntry } from './store.js';
+export interface WriterDeps {
+    client: QdrantClient;
+    embed: (texts: string[], timeoutMs?: number) => Promise<Float32Array[]>;
+}
+/** Time for embedding this many texts: a 30 s floor, and more for a long batch (a long record is many chunks). */
+export declare const embedBudget: (n: number) => number;
+/** A short-term entry's item id in the index (its long-term episode, if consolidation keeps it, is `ep:<uid>`). */
+export declare const stmItem: (uid: string) => string;
+/** The raw key of a captured request: what it asked and the start of what was done — ~150 words. */
+export declare function rawKey(e: Pick<StmEntry, 'request' | 'reply'>): string;
+/** A captured request's own title, its first line: the conversation's title says where it was, not what it is (one
+ *  long conversation's requests would all read alike, and match alike). */
+export declare function stmTitle(e: Pick<StmEntry, 'request' | 'title'>): string;
+export declare function writeStm(d: WriterDeps, entries: StmEntry[]): Promise<void>;
+/** A cue's point number: the k-th cue is -(k + 1) — below the key (0) and the children (1, 2, …), however many of
+ *  either there are, so the two never meet. */
+export declare const cuePoint: (k: number) => number;
+/** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
+ *  `eventAt`, or a date an agent searches by) — null for anything else. */
+export declare function dateSpan(d: string | undefined): {
+    from: number;
+    to: number;
+} | null;
+/** Only the cues of these items, written again (the key and the children stay as they are). */
+export declare function writeCues(d: WriterDeps, items: MemoryItem[]): Promise<void>;
+/** Embed and write long-term items: their old points out, the key point, the children and the cues in. Cues are
+ *  embedded as documents titled by the item (how they were measured); the item, not the cue, is what comes back. */
+/** A summary this many chunks long (~380 words) is also chunked: its key vector reads only its start. */
+export declare const LONG_SUMMARY_CHUNKS = 5;
+export declare function writeLtm(d: WriterDeps, items: MemoryItem[]): Promise<void>;
+/** The keys of an item's concepts, one per concept (text.ts conceptKey). */
+export declare function conceptKeys(names: readonly string[] | undefined): string[];
+/** Only the concepts of these items, written again (their keys on the key point). */
+export declare function writeConcepts(d: WriterDeps, items: MemoryItem[]): Promise<void>;

package/dist/core/writer.js

@@ -0,0 +1,146 @@
+/**
+ * How memory reaches the index: points built, embedded in batches and written to Qdrant — both tiers.
+ *
+ *   - a record (a short-term entry): ONE key point, its start as the key, and its whole text in chunks: no model,
+ *     searchable the moment it is added;
+ *   - a long-term item: a key point (content title + ~150-word summary, BM25 over title, summary, body and files) and
+ *     its ~96-word children (small-to-big: searched, never shown — the item is what comes back), its cues, and the keys
+ *     of its concepts on the key point (the concept graph's links: graph.ts).
+ */
+import { basename } from 'node:path';
+import { docText } from './embedder.js';
+import { episodeBody } from './episodes.js';
+import { pointId } from './qdrant-client.js';
+import { children, conceptKey, cut, docSparse, firstWords } from './text.js';
+/** Time for embedding this many texts: a 30 s floor, and more for a long batch (a long record is many chunks). */
+export const embedBudget = (n) => Math.max(30_000, n * 150);
+/** A short-term entry's item id in the index (its long-term episode, if consolidation keeps it, is `ep:<uid>`). */
+export const stmItem = (uid) => `stm:${uid}`;
+/** The raw key of a captured request: what it asked and the start of what was done — ~150 words. */
+export function rawKey(e) {
+    return firstWords(`${e.request}\n${e.reply}`, 150);
+}
+/** A captured request's own title, its first line: the conversation's title says where it was, not what it is (one
+ *  long conversation's requests would all read alike, and match alike). */
+export function stmTitle(e) {
+    const first = e.request.split('\n').map((l) => l.trim()).find(Boolean) ?? '';
+    return cut(first, 100) || e.title;
+}
+export async function writeStm(d, entries) {
+    if (!entries.length)
+        return;
+    // The raw key (its start), and the whole body in chunks: every word of a captured request can be found.
+    const plan = entries.map((e) => { const body = episodeBody(e); return { e, body, kids: children(body) }; });
+    const texts = plan.flatMap(({ e, kids }) => [docText(stmTitle(e), rawKey(e)), ...kids.map((k) => docText(stmTitle(e), k))]);
+    const vecs = await d.embed(texts, embedBudget(texts.length));
+    const points = [];
+    let at = 0;
+    for (const { e, body, kids } of plan) {
+        const common = { item: stmItem(e.uid), tier: 'stm', status: 'active', kind: 'episode', scope: 'project', project: e.project, session: e.session, ts: e.ts, synthetic: e.synthetic };
+        points.push({
+            id: pointId(stmItem(e.uid), 0),
+            vector: { dense: Array.from(vecs[at]), bm25: docSparse(`${stmTitle(e)}\n${body}`) },
+            payload: { ...common, part: 'key', engine: e.engine, cwd: e.cwd ?? '', where: e.title, title: stmTitle(e), summary: rawKey(e), body, files: e.files, importance: 0, ...(e.record ? { record: e.record } : {}) },
+        });
+        kids.forEach((_, k) => points.push({ id: pointId(stmItem(e.uid), k + 1), vector: { dense: Array.from(vecs[at + 1 + k]) }, payload: { ...common, part: 'child' } }));
+        at += 1 + kids.length;
+    }
+    await d.client.deleteItems(entries.map((e) => stmItem(e.uid)));
+    for (let i = 0; i < points.length; i += 256)
+        await d.client.upsert(points.slice(i, i + 256), true);
+}
+/** A cue's point number: the k-th cue is -(k + 1) — below the key (0) and the children (1, 2, …), however many of
+ *  either there are, so the two never meet. */
+export const cuePoint = (k) => -(k + 1);
+/** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
+ *  `eventAt`, or a date an agent searches by) — null for anything else. */
+export function dateSpan(d) {
+    const m = /^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$/.exec(d ?? '');
+    if (!m)
+        return null;
+    const [y, mo, dd] = [Number(m[1]), m[2] ? Number(m[2]) - 1 : null, m[3] ? Number(m[3]) : null];
+    if (mo === null)
+        return { from: Date.UTC(y, 0, 1), to: Date.UTC(y + 1, 0, 1) - 1 };
+    if (dd === null)
+        return { from: Date.UTC(y, mo, 1), to: Date.UTC(y, mo + 1, 1) - 1 };
+    return { from: Date.UTC(y, mo, dd), to: Date.UTC(y, mo, dd + 1) - 1 };
+}
+/** What every point of a long-term item carries: what recall and the filters select on — for an observation, the time
+ *  it happened too (memory_search's `from`/`to`): from its date, or from when what it tells began to when it ended. */
+function ltmPayload(it) {
+    const spans = [it.eventAt, it.startAt, it.endAt].map(dateSpan).filter((s) => !!s);
+    const span = spans.length ? { from: Math.min(...spans.map((s) => s.from)), to: Math.max(...spans.map((s) => s.to)) } : null;
+    return {
+        item: it.uid, tier: 'ltm', status: it.status, kind: it.kind, scope: it.scope, project: it.project,
+        session: it.session ?? '', ts: it.ts, synthetic: false, ...(span ? { eventFrom: span.from, eventTo: span.to } : {}),
+    };
+}
+/** Only the cues of these items, written again (the key and the children stay as they are). */
+export async function writeCues(d, items) {
+    const live = items.filter((it) => it.status === 'active');
+    const texts = live.flatMap((it) => (it.cues ?? []).map((c) => docText(it.title, c)));
+    if (!texts.length) {
+        await d.client.deleteParts(live.map((it) => it.uid), 'cue');
+        return;
+    }
+    const vecs = await d.embed(texts, embedBudget(texts.length));
+    const points = [];
+    let n = 0;
+    for (const it of live) {
+        (it.cues ?? []).forEach((_, k) => points.push({ id: pointId(it.uid, cuePoint(k)), vector: { dense: Array.from(vecs[n++]) }, payload: { ...ltmPayload(it), part: 'cue' } }));
+    }
+    await d.client.deleteParts(live.map((it) => it.uid), 'cue');
+    for (let i = 0; i < points.length; i += 256)
+        await d.client.upsert(points.slice(i, i + 256), true);
+}
+/** Embed and write long-term items: their old points out, the key point, the children and the cues in. Cues are
+ *  embedded as documents titled by the item (how they were measured); the item, not the cue, is what comes back. */
+/** A summary this many chunks long (~380 words) is also chunked: its key vector reads only its start. */
+export const LONG_SUMMARY_CHUNKS = 5;
+export async function writeLtm(d, items) {
+    const live = items.filter((it) => it.status === 'active');
+    if (!live.length) {
+        await d.client.deleteItems(items.map((it) => it.uid));
+        return;
+    }
+    const texts = [];
+    const plan = [];
+    for (const it of live) {
+        // Every chunk of the body — and of a summary too long for one key vector (the embedder reads its start).
+        const kids = [...children(it.body), ...(children(it.summary).length >= LONG_SUMMARY_CHUNKS ? children(it.summary) : [])];
+        const cues = it.cues ?? [];
+        plan.push({ it, kids, cues, at: texts.length });
+        texts.push(docText(it.title, it.summary), ...kids.map((k) => docText(it.title, k)), ...cues.map((c) => docText(it.title, c)));
+    }
+    const vecs = await d.embed(texts, embedBudget(texts.length));
+    const points = [];
+    for (const { it, kids, cues, at } of plan) {
+        const common = ltmPayload(it);
+        points.push({
+            id: pointId(it.uid, 0),
+            vector: { dense: Array.from(vecs[at]), bm25: docSparse(`${it.title}\n${it.summary}\n${it.body}\n${it.files.map((f) => basename(f)).join(' ')}`) },
+            payload: {
+                ...common, part: 'key', engine: it.engine, cwd: it.cwd ?? '', where: it.where, title: it.title, summary: it.summary,
+                body: it.body, files: it.files, importance: it.importance, appliesWhen: it.appliesWhen,
+                related: it.related ?? [], concepts: conceptKeys(it.concepts),
+                ...(it.eventAt ? { eventAt: it.eventAt } : {}), ...(it.startAt ? { startAt: it.startAt } : {}), ...(it.endAt ? { endAt: it.endAt } : {}),
+                ...(it.phase ? { phase: it.phase } : {}), ...(it.about ? { about: it.about } : {}),
+            },
+        });
+        kids.forEach((_, k) => points.push({ id: pointId(it.uid, k + 1), vector: { dense: Array.from(vecs[at + 1 + k]) }, payload: { ...common, part: 'child' } }));
+        cues.forEach((_, k) => points.push({ id: pointId(it.uid, cuePoint(k)), vector: { dense: Array.from(vecs[at + 1 + kids.length + k]) }, payload: { ...common, part: 'cue' } }));
+    }
+    // Remote embedding/validation can fail: keep the old searchable item until every replacement vector exists.
+    await d.client.deleteItems(items.map((it) => it.uid));
+    for (let i = 0; i < points.length; i += 256)
+        await d.client.upsert(points.slice(i, i + 256), true);
+}
+/** The keys of an item's concepts, one per concept (text.ts conceptKey). */
+export function conceptKeys(names) {
+    return [...new Set((names ?? []).map(conceptKey).filter(Boolean))];
+}
+/** Only the concepts of these items, written again (their keys on the key point). */
+export async function writeConcepts(d, items) {
+    for (const it of items.filter((x) => x.status === 'active'))
+        await d.client.setPayload([it.uid], { concepts: conceptKeys(it.concepts) }, true);
+}

package/dist/importers/files.d.ts

@@ -0,0 +1,31 @@
+import { type MemoryRecord } from '../records.js';
+/** The extensions read by default: text formats only. */
+export declare const TEXT_EXTENSIONS: string[];
+export interface FilesOptions {
+    /** Extensions to read, with or without the dot. Default: TEXT_EXTENSIONS. */
+    include?: string[];
+    /** Folder names never entered (dot folders never are). Default: `node_modules`. */
+    skipFolders?: string[];
+    /** Leave out a file by its path inside the folder. */
+    exclude?: (path: string) => boolean;
+    /** Longer files are cut into parts of at most this many characters. Default 20,000. */
+    maxChars?: number;
+    /** Larger files are not read (reported through `onSkip`). Default 64 MiB. */
+    maxBytes?: number;
+    collection?: string;
+    /** Default: the folder's name. */
+    source?: string;
+    /** Default `note`. */
+    kind?: string;
+    /** Called for each file left out, with why. */
+    onSkip?: (path: string, reason: string) => void;
+}
+type Scalar = string | number | boolean | null;
+/** YAML (`---`) or TOML (`+++`) front matter, flat fields only: scalars, lists (joined with `, `) and block text. */
+export declare function frontMatter(text: string): {
+    fields: Record<string, Scalar>;
+    body: string;
+};
+/** Records from the text files in a folder and its subfolders. */
+export declare function fromFiles(dir: string, opts?: FilesOptions): AsyncIterable<MemoryRecord>;
+export {};

package/dist/importers/files.js

@@ -0,0 +1,157 @@
+/**
+ * A folder of text files: notes, documents, saved pages. Every file whose extension is in `include` becomes a record,
+ * read one at a time; dot folders, dot files and `node_modules` are passed by.
+ *
+ * id: the path inside the folder (with `/`). title: the Markdown front matter's `title`, else the first `# heading`
+ * (an Org `#+TITLE:`, an RST underlined title, an HTML `<title>` or `<h1>`), else the file's name. text: the file, front
+ * matter taken off, HTML reduced to text. at: the front matter's `date`, else when the file was last changed. The other
+ * front-matter fields go into metadata (`author` also to the record's author). A file longer than `maxChars` is cut
+ * into parts with ids `path#1`, `path#2`, … sharing the thread `path`.
+ */
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { basename, extname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { timeOf } from '../records.js';
+import { decodeBytes, htmlTitle, htmlToText, record, splitText } from './text.js';
+/** The extensions read by default: text formats only. */
+export const TEXT_EXTENSIONS = ['.md', '.markdown', '.mdx', '.txt', '.text', '.html', '.htm', '.xhtml', '.rst', '.org', '.adoc',
+    '.asciidoc', '.textile', '.wiki', '.tex'];
+function scalar(raw) {
+    const v = raw.trim();
+    if (/^(["']).*\1$/s.test(v))
+        return v.slice(1, -1).replace(/\\"/g, '"').replace(/''/g, "'");
+    if (/^(true|false)$/i.test(v))
+        return v.toLowerCase() === 'true';
+    if (/^(null|~)$/i.test(v) || v === '')
+        return null;
+    if (/^-?(0|[1-9]\d{0,14})(\.\d+)?$/.test(v))
+        return Number(v);
+    return v;
+}
+function listValue(items) {
+    return items.map((i) => scalar(i)).filter((i) => i !== null && i !== '').join(', ');
+}
+/** YAML (`---`) or TOML (`+++`) front matter, flat fields only: scalars, lists (joined with `, `) and block text. */
+export function frontMatter(text) {
+    const m = /^(---|\+\+\+)[ \t]*\r?\n([\s\S]*?)\r?\n(?:---|\+\+\+|\.\.\.)[ \t]*(?:\r?\n|$)/.exec(text);
+    if (!m)
+        return { fields: {}, body: text };
+    const toml = m[1] === '+++';
+    const fields = {};
+    const lines = m[2].split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const kv = toml ? /^([^\s=#[][^=]*?)\s*=\s*(.*)$/.exec(lines[i]) : /^([^\s:#-][^:]*?)\s*:(?:\s+(.*)|\s*)$/.exec(lines[i]);
+        if (!kv)
+            continue;
+        const key = kv[1].trim().replace(/^["']|["']$/g, '');
+        const value = (kv[2] ?? '').trim();
+        if (value.startsWith('[') && value.endsWith(']')) {
+            fields[key] = listValue(value.slice(1, -1).split(','));
+        }
+        else if (!toml && /^[|>][-+]?$/.test(value)) {
+            const block = [];
+            while (i + 1 < lines.length && (/^\s/.test(lines[i + 1]) || !lines[i + 1].trim()))
+                block.push(lines[++i].trim());
+            fields[key] = block.join(value.startsWith('|') ? '\n' : ' ').trim();
+        }
+        else if (!toml && !value) {
+            const items = [];
+            while (i + 1 < lines.length && /^\s*-\s+/.test(lines[i + 1]))
+                items.push(lines[++i].replace(/^\s*-\s+/, ''));
+            while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1]))
+                i++; // a nested map: not kept
+            if (items.length)
+                fields[key] = listValue(items);
+        }
+        else {
+            fields[key] = scalar(value.replace(/\s+#.*$/, ''));
+        }
+    }
+    return { fields, body: text.slice(m[0].length) };
+}
+function titleOf(ext, body, raw) {
+    if (ext === '.html' || ext === '.htm' || ext === '.xhtml')
+        return htmlTitle(raw);
+    if (ext === '.org')
+        return /^#\+title:[ \t]*(.+)$/im.exec(body)?.[1].trim() || /^\*+[ \t]+(.+)$/m.exec(body)?.[1].trim();
+    if (ext === '.rst')
+        return /^(?:([=\-~^"'`#*+])\1+\r?\n)?[ \t]*(\S[^\n]*?)[ \t]*\r?\n([=\-~^"'`#*+])\3{2,}[ \t]*$/m.exec(body)?.[2];
+    if (ext === '.adoc' || ext === '.asciidoc')
+        return /^=[ \t]+(.+)$/m.exec(body)?.[1].trim();
+    const atx = /^#[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/m.exec(body)?.[1].trim();
+    if (atx)
+        return atx;
+    return /^(\S[^\n]*?)[ \t]*\r?\n=+[ \t]*$/m.exec(body)?.[1]; // a Setext `Title\n=====`
+}
+async function* walk(root, rel, o) {
+    const entries = await readdir(join(root, rel), { withFileTypes: true });
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const e of entries) {
+        if (e.name.startsWith('.'))
+            continue;
+        const path = rel ? `${rel}/${e.name}` : e.name;
+        if (e.isDirectory()) {
+            if (!o.skipFolders.has(e.name))
+                yield* walk(root, path, o);
+        }
+        else if (e.isFile() && o.include.has(extname(e.name).toLowerCase())) {
+            yield path;
+        }
+    }
+}
+function decodeFile(bytes) {
+    if (bytes[0] === 0xff && bytes[1] === 0xfe)
+        return new TextDecoder('utf-16le').decode(bytes);
+    if (bytes[0] === 0xfe && bytes[1] === 0xff)
+        return new TextDecoder('utf-16be').decode(bytes);
+    if (bytes.subarray(0, 8192).includes(0))
+        return null; // binary
+    return decodeBytes(bytes);
+}
+/** Records from the text files in a folder and its subfolders. */
+export async function* fromFiles(dir, opts = {}) {
+    const root = resolve(dir);
+    const include = new Set((opts.include ?? TEXT_EXTENSIONS).map((e) => (e.startsWith('.') ? e : `.${e}`).toLowerCase()));
+    const skipFolders = new Set(opts.skipFolders ?? ['node_modules']);
+    const maxChars = opts.maxChars ?? 20_000;
+    const maxBytes = opts.maxBytes ?? 64 * 1024 * 1024;
+    const source = opts.source ?? basename(root);
+    for await (const path of walk(root, '', { include, skipFolders })) {
+        if (opts.exclude?.(path))
+            continue;
+        const abs = join(root, path);
+        const info = await stat(abs);
+        if (info.size > maxBytes) {
+            opts.onSkip?.(path, `larger than ${maxBytes} bytes`);
+            continue;
+        }
+        const raw = decodeFile(await readFile(abs));
+        if (raw === null) {
+            opts.onSkip?.(path, 'not text');
+            continue;
+        }
+        const ext = extname(path).toLowerCase();
+        const isHtml = ext === '.html' || ext === '.htm' || ext === '.xhtml';
+        const { fields, body } = isHtml ? { fields: {}, body: raw } : frontMatter(raw);
+        const fmTitle = typeof fields.title === 'string' || typeof fields.title === 'number' ? String(fields.title).trim() : '';
+        const title = fmTitle || titleOf(ext, body, raw) || basename(path, extname(path));
+        const text = isHtml ? htmlToText(raw) : body.replace(/\r\n?/g, '\n').trim();
+        const dated = typeof fields.date === 'string' || typeof fields.date === 'number' ? timeOf(String(fields.date)) : undefined;
+        const at = dated ?? info.mtimeMs;
+        const author = typeof fields.author === 'string' ? fields.author : undefined;
+        const metadata = Object.fromEntries(Object.entries(fields).filter(([k]) => k !== 'title'));
+        const base = { id: path, text, title, kind: opts.kind ?? 'note', source, collection: opts.collection, author,
+            at: Math.round(at), url: pathToFileURL(abs).href };
+        if (!text && !title)
+            continue;
+        const parts = splitText(text, maxChars);
+        if (parts.length === 1) {
+            yield record({ ...base, metadata });
+            continue;
+        }
+        for (let i = 0; i < parts.length; i++) {
+            yield record({ ...base, id: `${path}#${i + 1}`, thread: path, text: parts[i], title: `${title} (${i + 1}/${parts.length})`,
+                metadata: { ...metadata, part: i + 1, parts: parts.length } });
+        }
+    }
+}

package/dist/importers/git.d.ts

@@ -0,0 +1,30 @@
+import type { MemoryRecord } from '../records.js';
+export interface GitOptions {
+    /** Only commits after this (anything `git log --since` reads, a Date, or milliseconds). */
+    since?: string | Date | number;
+    /** Only commits before this. */
+    until?: string | Date | number;
+    /** The branch, tag or range to read (`main`, `v1.0..v2.0`). Default: HEAD. `rev` is the same. */
+    branch?: string;
+    rev?: string;
+    /** Every branch and tag. */
+    all?: boolean;
+    /** At most this many commits. */
+    maxCount?: number;
+    /** Only commits that changed these paths. */
+    paths?: string[];
+    /** Leave out merge commits. */
+    noMerges?: boolean;
+    /** At most this many changed paths listed in a record's text. Default 50. */
+    maxFiles?: number;
+    /** Default: the repository's name. */
+    thread?: string;
+    source?: string;
+    collection?: string;
+    /** Where to open a commit, e.g. `(sha) => \`https://example.org/repo/commit/${sha}\``. */
+    url?: (sha: string) => string;
+    /** The git program. Default `git`. */
+    git?: string;
+}
+/** Records from a repository's commits, newest first (as `git log` gives them). */
+export declare function fromGit(repoDir: string, opts?: GitOptions): AsyncIterable<MemoryRecord>;

package/dist/importers/git.js

@@ -0,0 +1,118 @@
+/**
+ * A Git repository's history: one record a commit, read from `git log` as it runs (never the whole log at once).
+ * Each commit is framed by markers carrying a random token made for the run, so no message text can be taken for one.
+ *
+ * id: the full sha. title: the subject. text: the whole message, then `Files: …` (the paths it changed, at most
+ * `maxFiles`). author: the author's name. at: the author date. kind `commit`; thread: `opts.thread` or the repository's
+ * name; source: `opts.source` or the repository's name; metadata { sha, email, parents }.
+ */
+import { spawn } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import { basename, resolve } from 'node:path';
+import { record } from './text.js';
+function when(t) {
+    return t instanceof Date ? t.toISOString() : typeof t === 'number' ? new Date(t).toISOString() : t;
+}
+/** A path as git writes it when it has unusual characters: `"a\tb\303\251"` back to its text. */
+function unquote(path) {
+    if (!(path.length > 1 && path.startsWith('"') && path.endsWith('"')))
+        return path;
+    const bytes = [];
+    const s = path.slice(1, -1);
+    const esc = { a: 7, b: 8, t: 9, n: 10, v: 11, f: 12, r: 13, '"': 34, '\\': 92 };
+    for (let i = 0; i < s.length; i++) {
+        if (s[i] === '\\' && /[0-7]{3}/.test(s.slice(i + 1, i + 4))) {
+            bytes.push(parseInt(s.slice(i + 1, i + 4), 8));
+            i += 3;
+        }
+        else if (s[i] === '\\' && s[i + 1] in esc) {
+            bytes.push(esc[s[++i]]);
+        }
+        else {
+            bytes.push(...Buffer.from(s[i], 'utf8'));
+        }
+    }
+    return Buffer.from(bytes).toString('utf8');
+}
+/** Records from a repository's commits, newest first (as `git log` gives them). */
+export async function* fromGit(repoDir, opts = {}) {
+    const name = basename(resolve(repoDir));
+    const token = randomBytes(16).toString('hex');
+    const start = `\u001e${token}\n`;
+    const end = `\u001f${token}`;
+    const args = ['-c', 'core.quotePath=false', '-c', 'log.showSignature=false', 'log', '--encoding=UTF-8', '--no-color',
+        '--no-ext-diff', '--name-only', `--format=%x1e${token}%n%H%x00%an%x00%ae%x00%aI%x00%P%x00%B%x1f${token}`];
+    if (opts.maxCount !== undefined)
+        args.push(`--max-count=${Math.max(0, Math.floor(opts.maxCount))}`);
+    if (opts.since !== undefined)
+        args.push(`--since=${when(opts.since)}`);
+    if (opts.until !== undefined)
+        args.push(`--until=${when(opts.until)}`);
+    if (opts.noMerges)
+        args.push('--no-merges');
+    if (opts.all)
+        args.push('--all');
+    const rev = opts.rev ?? opts.branch;
+    if (rev) {
+        if (rev.startsWith('-'))
+            throw new Error(`not a revision: ${rev}`);
+        args.push(rev);
+    }
+    args.push('--', ...(opts.paths ?? []));
+    const maxFiles = opts.maxFiles ?? 50;
+    const child = spawn(opts.git ?? 'git', args, { cwd: repoDir, stdio: ['ignore', 'pipe', 'pipe'] });
+    let stderr = '';
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (d) => {
+        if (stderr.length < 4000)
+            stderr += d;
+    });
+    const exited = new Promise((done, fail) => {
+        child.on('error', fail);
+        child.on('close', (code) => done(code));
+    });
+    exited.catch(() => undefined); // read below; this only keeps an early stop from leaving it unhandled
+    const toRecord = (chunk) => {
+        const close = chunk.lastIndexOf(end);
+        const head = close >= 0 ? chunk.slice(0, close) : chunk;
+        const [sha, author, email, date, parents, ...message] = head.split('\u0000');
+        if (!sha || !/^[0-9a-f]{7,64}$/.test(sha))
+            return null;
+        const body = message.join('\u0000').replace(/\r\n?/g, '\n').trim();
+        const subject = (body.split(/\n\s*\n/)[0] ?? '').replace(/\s*\n\s*/g, ' ').trim();
+        const files = close >= 0 ? chunk.slice(close + end.length).split('\n').filter((l) => l.trim()).map(unquote) : [];
+        const listed = files.slice(0, maxFiles).join(', ') + (files.length > maxFiles ? `, … and ${files.length - maxFiles} more` : '');
+        const text = [body, files.length ? `Files: ${listed}` : ''].filter(Boolean).join('\n\n');
+        return record({ id: sha, title: subject || sha.slice(0, 12), text, kind: 'commit', author, at: date || undefined,
+            thread: opts.thread ?? name, source: opts.source ?? name, collection: opts.collection, url: opts.url?.(sha),
+            metadata: { sha, email, parents } });
+    };
+    let buffer = '';
+    try {
+        child.stdout.setEncoding('utf8');
+        for await (const piece of child.stdout) {
+            buffer += piece;
+            let next = buffer.indexOf(start, 1);
+            while (next > 0) {
+                const r = toRecord(buffer.slice(buffer.startsWith(start) ? start.length : 0, next));
+                if (r)
+                    yield r;
+                buffer = buffer.slice(next);
+                next = buffer.indexOf(start, 1);
+            }
+        }
+        if (buffer.startsWith(start)) {
+            const r = toRecord(buffer.slice(start.length));
+            if (r)
+                yield r;
+        }
+    }
+    finally {
+        if (child.exitCode === null && child.signalCode === null)
+            child.kill();
+    }
+    const code = await exited;
+    if (code !== 0 && !/does not have any commits|bad default revision 'HEAD'/.test(stderr)) {
+        throw new Error(`git log failed (${code}) in ${repoDir}: ${stderr.trim()}`);
+    }
+}

package/dist/importers/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Importers: records from what people already have — JSON Lines, a folder of text files, a Git history, a Slack
+ * export, an mbox or `.eml` mailbox. Each one streams: records come one at a time as the source is read, never the
+ * whole source in memory, so `for await (const batch of batches(fromMbox(path), 200)) await memory.add(batch)` works
+ * on a mailbox of any size.
+ */
+export { fromJsonl, type JsonlMap, type JsonlOptions } from './jsonl.js';
+export { fromFiles, frontMatter, TEXT_EXTENSIONS, type FilesOptions } from './files.js';
+export { fromGit, type GitOptions } from './git.js';
+export { fromSlackExport, slackText, SLACK_SKIPPED_SUBTYPES, type SlackOptions } from './slack.js';
+export { fromMbox, fromEml, decodeEncodedWords, type MboxOptions } from './mbox.js';
+export { htmlToText, decodeEntities, decodeBytes, splitText } from './text.js';
+/** Items in arrays of `size` (the last may be shorter), as they come. */
+export declare function batches<T>(it: AsyncIterable<T> | Iterable<T>, size: number): AsyncIterable<T[]>;

package/dist/importers/index.js

@@ -0,0 +1,26 @@
+/**
+ * Importers: records from what people already have — JSON Lines, a folder of text files, a Git history, a Slack
+ * export, an mbox or `.eml` mailbox. Each one streams: records come one at a time as the source is read, never the
+ * whole source in memory, so `for await (const batch of batches(fromMbox(path), 200)) await memory.add(batch)` works
+ * on a mailbox of any size.
+ */
+export { fromJsonl } from './jsonl.js';
+export { fromFiles, frontMatter, TEXT_EXTENSIONS } from './files.js';
+export { fromGit } from './git.js';
+export { fromSlackExport, slackText, SLACK_SKIPPED_SUBTYPES } from './slack.js';
+export { fromMbox, fromEml, decodeEncodedWords } from './mbox.js';
+export { htmlToText, decodeEntities, decodeBytes, splitText } from './text.js';
+/** Items in arrays of `size` (the last may be shorter), as they come. */
+export async function* batches(it, size) {
+    const n = Math.max(1, Math.floor(size));
+    let batch = [];
+    for await (const item of it) {
+        batch.push(item);
+        if (batch.length >= n) {
+            yield batch;
+            batch = [];
+        }
+    }
+    if (batch.length)
+        yield batch;
+}

package/dist/importers/jsonl.d.ts

@@ -0,0 +1,20 @@
+import { type MemoryRecord } from '../records.js';
+/** Turns one parsed line into a record, or null to leave it out. */
+export type JsonlMap = (obj: unknown, lineNumber: number) => MemoryRecord | null | undefined;
+export interface JsonlOptions {
+    map?: JsonlMap;
+    /** Called for each line left out, with why. */
+    onSkip?: (line: string, reason: string, lineNumber: number) => void;
+    /** Filled in as the file is read: lines read, records given, lines skipped. */
+    stats?: {
+        lines: number;
+        records: number;
+        skipped: number;
+    };
+    /** For records that have none. */
+    source?: string;
+    collection?: string;
+    kind?: string;
+}
+/** Records from a JSON Lines file; `map` may be given here or in the options. */
+export declare function fromJsonl(path: string, map?: JsonlMap | JsonlOptions, opts?: JsonlOptions): AsyncIterable<MemoryRecord>;

package/dist/importers/jsonl.js

@@ -0,0 +1,65 @@
+/**
+ * JSON Lines: one JSON object per line, read a line at a time. With no `map`, each line is a record as it is
+ * (`{ "id": "…", "text": "…", "title": …, "at": … }`); with one, `map(obj)` turns each object into a record, or into
+ * null to leave it out. Lines that are not JSON or not a record are skipped, counted and reported through `onSkip`.
+ */
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { invalidRecord } from '../records.js';
+import { record } from './text.js';
+/** Records from a JSON Lines file; `map` may be given here or in the options. */
+export async function* fromJsonl(path, map, opts = {}) {
+    const o = typeof map === 'function' ? { ...opts, map } : { ...map, ...opts };
+    const stats = o.stats ?? { lines: 0, records: 0, skipped: 0 };
+    const skip = (line, reason, n) => {
+        stats.skipped++;
+        o.onSkip?.(line, reason, n);
+    };
+    const lines = createInterface({ input: createReadStream(path, { encoding: 'utf8' }), crlfDelay: Infinity });
+    let n = 0;
+    for await (const raw of lines) {
+        n++;
+        const line = n === 1 ? raw.replace(/^/, '') : raw;
+        if (!line.trim())
+            continue;
+        stats.lines++;
+        let obj;
+        try {
+            obj = JSON.parse(line);
+        }
+        catch (e) {
+            skip(line, `not JSON: ${e.message}`, n);
+            continue;
+        }
+        let r;
+        if (o.map) {
+            try {
+                r = o.map(obj, n);
+            }
+            catch (e) {
+                skip(line, `map failed: ${e.message}`, n);
+                continue;
+            }
+            if (!r) {
+                skip(line, 'left out by map', n);
+                continue;
+            }
+        }
+        else {
+            if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+                skip(line, 'not an object', n);
+                continue;
+            }
+            const shaped = obj;
+            r = typeof shaped.id === 'number' ? { ...shaped, id: String(shaped.id) } : shaped;
+        }
+        const filled = record({ ...r, source: r.source ?? o.source, collection: r.collection ?? o.collection, kind: r.kind ?? o.kind });
+        const wrong = invalidRecord(filled);
+        if (wrong) {
+            skip(line, wrong, n);
+            continue;
+        }
+        stats.records++;
+        yield filled;
+    }
+}