memtrails 0.0.0-stage → 0.2.0

package/dist/importers/mbox.d.ts

@@ -0,0 +1,20 @@
+import { type MemoryRecord } from '../records.js';
+export interface MboxOptions {
+    /** Keep lines quoted from earlier messages (`> …`). */
+    keepQuoted?: boolean;
+    /** Keep the signature (what follows a `-- ` line). */
+    keepSignature?: boolean;
+    /** Default: the file's name without its extension. */
+    source?: string;
+    collection?: string;
+    /** A message's body past this many bytes is not read (its headers are). Default 32 MiB. */
+    maxMessageBytes?: number;
+    /** Called for each message left out (no subject and no text), with its place in the file. */
+    onSkip?: (reason: string, index: number) => void;
+}
+/** RFC 2047 encoded-words decoded; words side by side join with no space, and their bytes are read together. */
+export declare function decodeEncodedWords(s: string): string;
+/** Records from an mbox file, a message at a time. */
+export declare function fromMbox(path: string, opts?: MboxOptions): AsyncIterable<MemoryRecord>;
+/** The record of one `.eml` file (an RFC 5322 message). */
+export declare function fromEml(path: string, opts?: MboxOptions): AsyncIterable<MemoryRecord>;

package/dist/importers/mbox.js

@@ -0,0 +1,347 @@
+/**
+ * E-mail: an mbox file (RFC 4155: each message opens with a `From ` line; `>From ` in a body is unescaped), read a
+ * message at a time, or one `.eml` file. Headers are unfolded and their RFC 2047 encoded-words (Q and B, any charset)
+ * decoded; MIME parts are taken apart (text/plain preferred, else text/html reduced to text), base64 and
+ * quoted-printable decoded, and every charset the platform knows read as such.
+ *
+ * id: the Message-ID (else a hash of the message). title: the Subject. text: the body, with the names of attachments
+ * after it; lines quoted from earlier messages (`>`) and the signature (after `-- `) are left out unless `keepQuoted` /
+ * `keepSignature`, since the earlier message is a record of its own. author: the sender's name, else address. at: the
+ * Date. kind `email`. thread: the first id in References, else In-Reply-To, else the message's own id.
+ * metadata { from, to, cc, inReplyTo }.
+ */
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { basename, extname } from 'node:path';
+import { createInterface } from 'node:readline';
+import { timeOf } from '../records.js';
+import { decodeBytes, htmlToText, record } from './text.js';
+function entityOf(lines) {
+    const headers = new Map();
+    let i = 0;
+    let name = '';
+    for (; i < lines.length; i++) {
+        const line = lines[i];
+        if (line === '' || line === '\r')
+            break;
+        if (/^[ \t]/.test(line)) {
+            if (name)
+                headers.set(name, `${headers.get(name)}${line.replace(/^[ \t]+/, ' ')}`);
+            continue;
+        }
+        const colon = line.indexOf(':');
+        if (colon <= 0)
+            continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        if (headers.has(key)) {
+            name = ''; // a later copy of a header: the first is kept
+            continue;
+        }
+        name = key;
+        headers.set(key, line.slice(colon + 1).trim());
+    }
+    return { headers, body: lines.slice(i + 1) };
+}
+function qBytes(s) {
+    const out = [];
+    for (let i = 0; i < s.length; i++) {
+        if (s[i] === '_')
+            out.push(0x20);
+        else if (s[i] === '=' && /^[0-9A-Fa-f]{2}$/.test(s.slice(i + 1, i + 3))) {
+            out.push(parseInt(s.slice(i + 1, i + 3), 16));
+            i += 2;
+        }
+        else
+            out.push(s.charCodeAt(i) & 0xff);
+    }
+    return out;
+}
+/** RFC 2047 encoded-words decoded; words side by side join with no space, and their bytes are read together. */
+export function decodeEncodedWords(s) {
+    let out = '';
+    let last = 0;
+    let pending = null;
+    const flush = () => {
+        if (pending)
+            out += decodeBytes(Uint8Array.from(pending.bytes), pending.charset);
+        pending = null;
+    };
+    for (const m of s.matchAll(/=\?([^?\s]+)\?([bBqQ])\?([^?\s]*)\?=/g)) {
+        const gap = s.slice(last, m.index);
+        if (!pending || !/^\s*$/.test(gap)) {
+            flush();
+            out += gap;
+        }
+        const charset = m[1].replace(/\*.*$/, '').toLowerCase();
+        const bytes = m[2].toUpperCase() === 'B' ? [...Buffer.from(m[3], 'base64')] : qBytes(m[3]);
+        if (pending && pending.charset !== charset)
+            flush();
+        if (pending)
+            pending.bytes.push(...bytes);
+        else
+            pending = { charset, bytes };
+        last = m.index + m[0].length;
+    }
+    flush();
+    return out + s.slice(last);
+}
+/** A raw header as text: bytes sent unencoded read as UTF-8 (else Windows-1252), then encoded-words decoded. */
+function headerText(raw) {
+    if (!raw)
+        return '';
+    return decodeEncodedWords(decodeBytes(Buffer.from(raw, 'latin1'))).replace(/\s+/g, ' ').trim();
+}
+/** `type/sub; a=1; b="x y"` as its value and parameters (RFC 2231 continued and encoded ones joined and decoded). */
+function parseParams(raw) {
+    const pieces = [];
+    let cur = '';
+    let quoted = false;
+    for (const ch of raw ?? '') {
+        if (ch === '"')
+            quoted = !quoted;
+        if (ch === ';' && !quoted) {
+            pieces.push(cur);
+            cur = '';
+        }
+        else
+            cur += ch;
+    }
+    pieces.push(cur);
+    const value = (pieces.shift() ?? '').trim().toLowerCase();
+    const params = {};
+    const sections = {};
+    for (const p of pieces) {
+        const eq = p.indexOf('=');
+        if (eq <= 0)
+            continue;
+        const key = p.slice(0, eq).trim().toLowerCase();
+        let v = p.slice(eq + 1).trim();
+        if (v.startsWith('"') && v.endsWith('"') && v.length > 1)
+            v = v.slice(1, -1).replace(/\\(.)/g, '$1');
+        const ext = /^(.+?)\*(\d+)?(\*)?$/.exec(key);
+        if (ext)
+            (sections[ext[1]] ??= []).push({ n: Number(ext[2] ?? 0), encoded: ext[2] === undefined || !!ext[3], v });
+        else
+            params[key] = key === 'name' || key === 'filename' ? headerText(v) : v;
+    }
+    for (const [key, list] of Object.entries(sections)) {
+        list.sort((a, b) => a.n - b.n);
+        let charset = '';
+        const bytes = [];
+        for (const [i, s] of list.entries()) {
+            let v = s.v;
+            if (s.encoded && i === 0) {
+                const m = /^([^']*)'[^']*'(.*)$/.exec(v);
+                if (m)
+                    [charset, v] = [m[1], m[2]];
+            }
+            if (s.encoded)
+                bytes.push(...Buffer.from(v.replace(/%([0-9A-Fa-f]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16))), 'latin1'));
+            else
+                bytes.push(...Buffer.from(v, 'latin1'));
+        }
+        params[key] = decodeBytes(Uint8Array.from(bytes), charset);
+    }
+    return { value, params };
+}
+function quotedPrintable(lines) {
+    let s = '';
+    for (const [i, raw] of lines.entries()) {
+        const line = raw.replace(/[ \t\r]+$/, '');
+        if (line.endsWith('='))
+            s += line.slice(0, -1);
+        else
+            s += i < lines.length - 1 ? `${line}\n` : line;
+    }
+    return Buffer.from(s.replace(/=([0-9A-Fa-f]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16))), 'latin1');
+}
+function bodyBytes(e) {
+    const cte = (e.headers.get('content-transfer-encoding') ?? '').trim().toLowerCase();
+    if (cte === 'base64')
+        return Buffer.from(e.body.join(''), 'base64');
+    if (cte === 'quoted-printable')
+        return quotedPrintable(e.body);
+    return Buffer.from(e.body.join('\n'), 'latin1');
+}
+/** `format=flowed` (RFC 3676) text with its soft line breaks joined. */
+function unflow(text, delsp) {
+    const out = [];
+    let cur = null;
+    for (const raw of text.split('\n')) {
+        const quote = /^>+/.exec(raw)?.[0] ?? '';
+        let line = raw.slice(quote.length);
+        if (line.startsWith(' '))
+            line = line.slice(1);
+        const soft = line.endsWith(' ') && line !== '-- ';
+        if (soft && delsp)
+            line = line.slice(0, -1);
+        cur = cur === null ? (quote ? `${quote} ${line}` : line) : cur + line;
+        if (!soft) {
+            out.push(cur);
+            cur = null;
+        }
+    }
+    if (cur !== null)
+        out.push(cur);
+    return out.join('\n');
+}
+function clean(text, html, opts) {
+    let lines = text.split('\n');
+    if (!opts.keepSignature) {
+        const sig = lines.findIndex((l) => l === '-- ' || (html && l === '--'));
+        if (sig >= 0)
+            lines = lines.slice(0, sig);
+    }
+    if (!opts.keepQuoted)
+        lines = lines.filter((l) => !/^ ?>/.test(l));
+    return lines.map((l) => (l === '-- ' ? l : l.trimEnd())).join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}
+function contentOf(e, opts, depth) {
+    const type = parseParams(e.headers.get('content-type') ?? 'text/plain');
+    const disposition = parseParams(e.headers.get('content-disposition'));
+    const name = disposition.params.filename ?? type.params.name;
+    const attached = disposition.value === 'attachment';
+    const mime = type.value || 'text/plain';
+    if (mime.startsWith('multipart/') && type.params.boundary && depth < 30) {
+        const kids = splitParts(e.body, type.params.boundary).map((k) => contentOf(k, opts, depth + 1));
+        const attachments = kids.flatMap((k) => k.attachments);
+        if (mime === 'multipart/alternative') {
+            const pick = kids.find((k) => k.text && !k.html) ?? kids.find((k) => k.text);
+            return { text: pick?.text ?? '', html: pick?.html ?? false, attachments: [...new Set(attachments)] };
+        }
+        const said = kids.filter((k) => k.text);
+        return { text: said.map((k) => k.text).join('\n\n'), html: said.length > 0 && said.every((k) => k.html), attachments };
+    }
+    if (mime === 'message/rfc822' && depth < 30) {
+        const inner = entityOf(bodyBytes(e).toString('latin1').split(/\r?\n/));
+        const c = contentOf(inner, opts, depth + 1);
+        const head = [['From', 'from'], ['Date', 'date'], ['Subject', 'subject']]
+            .map(([label, key]) => (inner.headers.has(key) ? `${label}: ${headerText(inner.headers.get(key))}` : ''))
+            .filter(Boolean);
+        return { text: [...head, '', c.text].join('\n').trim(), html: false, attachments: c.attachments };
+    }
+    if (!attached && (mime === 'text/plain' || mime === 'text/html' || mime === 'text/markdown')) {
+        const raw = decodeBytes(bodyBytes(e), type.params.charset).replace(/\r\n?/g, '\n');
+        if (mime === 'text/html')
+            return { text: clean(htmlToText(raw, { dropQuotes: !opts.keepQuoted }), true, opts), html: true, attachments: [] };
+        const flowed = type.params.format?.toLowerCase() === 'flowed';
+        const text = flowed ? unflow(raw, type.params.delsp?.toLowerCase() === 'yes') : raw;
+        return { text: clean(text, false, opts), html: false, attachments: [] };
+    }
+    return { text: '', html: false, attachments: name ? [name] : [] };
+}
+function splitParts(body, boundary) {
+    const open = `--${boundary}`;
+    const close = `--${boundary}--`;
+    const parts = [];
+    let cur = null;
+    for (const line of body) {
+        const t = line.trimEnd();
+        if (t === close)
+            break;
+        if (t === open) {
+            if (cur)
+                parts.push(cur);
+            cur = [];
+        }
+        else if (cur)
+            cur.push(line);
+    }
+    if (cur)
+        parts.push(cur);
+    return parts.map((p) => (p[0] === '' ? { headers: new Map(), body: p.slice(1) } : entityOf(p)));
+}
+function ids(raw) {
+    const v = raw ?? '';
+    const bracketed = [...v.matchAll(/<([^<>\s]+)>/g)].map((m) => m[1]);
+    return bracketed.length ? bracketed : v.split(/\s+/).filter((x) => x.includes('@'));
+}
+function personOf(from) {
+    const named = /^(.*?)\s*<([^<>]*)>\s*$/.exec(from);
+    if (named)
+        return named[1].replace(/^"(.*)"$/, '$1').trim() || named[2].trim();
+    const commented = /^\s*(\S+@\S+)\s*\((.+)\)\s*$/.exec(from);
+    return commented ? commented[2].trim() : from.trim();
+}
+/** One message's lines (raw bytes as Latin-1 characters, no line endings) as a record, or null when it says nothing. */
+function messageRecord(lines, fromLine, source, opts) {
+    while (lines.length && !lines[lines.length - 1].trim())
+        lines.pop();
+    const e = entityOf(lines);
+    const h = (key) => headerText(e.headers.get(key));
+    const content = contentOf(e, opts, 0);
+    const attachments = [...new Set(content.attachments)];
+    const text = [content.text, attachments.length ? `Attachments: ${attachments.join(', ')}` : ''].filter(Boolean).join('\n\n');
+    const subject = h('subject');
+    if (!text && !subject)
+        return null;
+    const from = h('from');
+    const date = h('date').replace(/\([^)]*\)/g, ' ').trim();
+    const at = timeOf(date) ?? timeOf(fromLine.replace(/^From \S+\s+/, '').trim());
+    const own = ids(e.headers.get('message-id'))[0];
+    const id = own ?? createHash('sha256').update([from, date, subject, text.slice(0, 4000)].join('\u0000')).digest('hex').slice(0, 32);
+    const inReplyTo = ids(e.headers.get('in-reply-to'))[0];
+    return record({
+        id, title: subject, text, kind: 'email', source, collection: opts.collection, author: from ? personOf(from) : undefined, at,
+        thread: ids(e.headers.get('references'))[0] ?? inReplyTo ?? id,
+        metadata: { from, to: h('to'), cc: h('cc'), inReplyTo },
+    });
+}
+/** Records from an mbox file, a message at a time. */
+export async function* fromMbox(path, opts = {}) {
+    const source = opts.source ?? basename(path, extname(path));
+    const max = opts.maxMessageBytes ?? 32 * 1024 * 1024;
+    // Latin-1 keeps every byte as one character: each part is decoded later in its own charset.
+    const lines = createInterface({ input: createReadStream(path, { encoding: 'latin1' }), crlfDelay: Infinity });
+    let msg = null;
+    let fromLine = '';
+    let size = 0;
+    let previousBlank = true;
+    let index = 0;
+    const finish = () => {
+        if (!msg)
+            return null;
+        const r = messageRecord(msg, fromLine, source, opts);
+        if (!r)
+            opts.onSkip?.('no subject and no text', index);
+        index++;
+        return r;
+    };
+    for await (const line of lines) {
+        // A new message: a `From ` line at the start or after an empty line, carrying a time (`From - Mon Jan 1 00:00:00 2024`).
+        if (line.startsWith('From ') && previousBlank && (msg === null || /\d:\d\d/.test(line))) {
+            const r = finish();
+            if (r)
+                yield r;
+            msg = [];
+            fromLine = line;
+            size = 0;
+            previousBlank = false;
+            continue;
+        }
+        previousBlank = line === '' || line === '\r';
+        if (msg === null) {
+            if (!line.trim())
+                continue;
+            msg = []; // a file that does not open with a `From ` line: one message from its first line
+        }
+        size += line.length + 1;
+        if (size > max)
+            continue;
+        msg.push(/^>+From /.test(line) ? line.slice(1) : line);
+    }
+    const r = finish();
+    if (r)
+        yield r;
+}
+/** The record of one `.eml` file (an RFC 5322 message). */
+export async function* fromEml(path, opts = {}) {
+    const lines = (await readFile(path)).toString('latin1').split(/\r?\n/);
+    const fromLine = lines[0]?.startsWith('From ') ? (lines.shift() ?? '') : '';
+    const r = messageRecord(lines, fromLine, opts.source ?? basename(path, extname(path)), opts);
+    if (r)
+        yield r;
+    else
+        opts.onSkip?.('no subject and no text', 0);
+}

package/dist/importers/slack.d.ts

@@ -0,0 +1,21 @@
+import type { MemoryRecord } from '../records.js';
+/** Message subtypes left out by default: events about the channel, not things said in it. */
+export declare const SLACK_SKIPPED_SUBTYPES: string[];
+export interface SlackOptions {
+    /** Only these conversations (by name or id). */
+    channels?: string[];
+    /** Default: the channel's name. */
+    collection?: string;
+    /** Default: the export folder's name. */
+    source?: string;
+    /** Subtypes to leave out. Default: SLACK_SKIPPED_SUBTYPES. */
+    skipSubtypes?: string[];
+    /** Leave out messages posted by bots and integrations. */
+    skipBots?: boolean;
+    /** The workspace's address (`https://example.slack.com`), for a link to each message. */
+    workspaceUrl?: string;
+}
+/** Slack's markup as plain text: mentions as names, channel links as `#name`, links as `label (url)`. */
+export declare function slackText(text: string, users: Map<string, string>, channels: Map<string, string>): string;
+/** Records from an unzipped Slack export folder. */
+export declare function fromSlackExport(dir: string, opts?: SlackOptions): AsyncIterable<MemoryRecord>;

package/dist/importers/slack.js

@@ -0,0 +1,134 @@
+/**
+ * A Slack workspace export, unzipped: `users.json`, `channels.json` (and `groups.json`, `mpims.json`, `dms.json` when
+ * present), and a folder a conversation holding one `YYYY-MM-DD.json` a day. Read a day at a time, in order.
+ *
+ * One record a message that says something. id: `channel id:ts`. text: the message with `<@U123>` as `@Name`,
+ * `<#C1|name>` as `#name`, `<url|label>` as `label (url)`, then the names of its files and attachments. author: the
+ * person's real name, else display name, else user name (a bot: its name). at: the message's ts. thread: `channel:ts`
+ * of the thread's first message (a message outside a thread starts its own). collection: `opts.collection` or the
+ * channel's name. kind `message`. Joins, leaves, topic changes and the like are left out.
+ */
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { basename, join, resolve } from 'node:path';
+import { record } from './text.js';
+/** Message subtypes left out by default: events about the channel, not things said in it. */
+export const SLACK_SKIPPED_SUBTYPES = ['channel_join', 'channel_leave', 'channel_topic', 'channel_purpose', 'channel_name',
+    'channel_archive', 'channel_unarchive', 'channel_convert_to_private', 'channel_convert_to_public', 'group_join', 'group_leave',
+    'group_topic', 'group_purpose', 'group_name', 'group_archive', 'group_unarchive', 'pinned_item', 'unpinned_item', 'bot_add',
+    'bot_remove', 'tombstone', 'reminder_add', 'joiner_notification', 'joiner_notification_for_inviter', 'message_deleted',
+    'sh_room_created', 'channel_canvas_updated'];
+async function readJson(path) {
+    try {
+        return JSON.parse((await readFile(path, 'utf8')).replace(/^/, ''));
+    }
+    catch (e) {
+        if (e.code === 'ENOENT')
+            return undefined;
+        throw new Error(`cannot read ${path}: ${e.message}`);
+    }
+}
+function personName(u) {
+    return [u.profile?.real_name, u.real_name, u.profile?.display_name, u.display_name, u.name].map((n) => n?.trim()).find(Boolean);
+}
+/** Slack's markup as plain text: mentions as names, channel links as `#name`, links as `label (url)`. */
+export function slackText(text, users, channels) {
+    return text
+        .replace(/<([^<>\n]*)>/g, (whole, inner) => {
+        const bar = inner.indexOf('|');
+        const target = bar >= 0 ? inner.slice(0, bar) : inner;
+        const label = bar >= 0 ? inner.slice(bar + 1) : '';
+        if (target.startsWith('@'))
+            return `@${users.get(target.slice(1)) ?? (label.replace(/^@/, '') || target.slice(1))}`;
+        if (target.startsWith('#'))
+            return `#${label || channels.get(target.slice(1)) || target.slice(1)}`;
+        if (target.startsWith('!')) {
+            const special = target.slice(1);
+            if (/^(here|channel|everyone)$/.test(special))
+                return `@${special}`;
+            return label || `@${special.replace(/\^.*$/, '')}`;
+        }
+        if (!target)
+            return whole;
+        if (target.startsWith('mailto:'))
+            return label || target.slice(7);
+        return label && label !== target ? `${label} (${target})` : target;
+    })
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&amp;/g, '&');
+}
+/** Records from an unzipped Slack export folder. */
+export async function* fromSlackExport(dir, opts = {}) {
+    const root = resolve(dir);
+    const source = opts.source ?? basename(root);
+    const skip = new Set(opts.skipSubtypes ?? SLACK_SKIPPED_SUBTYPES);
+    const users = new Map();
+    for (const u of (await readJson(join(root, 'users.json'))) ?? []) {
+        const n = personName(u);
+        if (u?.id && n)
+            users.set(u.id, n);
+    }
+    // Each conversation: its id, its name, and the folder its days are in (a direct message's folder is its id).
+    const conversations = [];
+    for (const [file, byId] of [['channels.json', false], ['groups.json', false], ['mpims.json', false], ['dms.json', true]]) {
+        for (const c of (await readJson(join(root, file))) ?? []) {
+            if (!c?.id)
+                continue;
+            const name = byId ? (c.members ?? []).map((m) => users.get(m) ?? m).join(', ') || c.id : c.name || c.id;
+            conversations.push({ id: c.id, name, folder: byId ? c.id : c.name || c.id });
+        }
+    }
+    if (!conversations.length) {
+        // No lists: every folder is a conversation named after it.
+        for (const e of (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !e.name.startsWith('.'))) {
+            conversations.push({ id: e.name, name: e.name, folder: e.name });
+        }
+    }
+    const channelNames = new Map(conversations.map((c) => [c.id, c.name]));
+    const wanted = opts.channels ? new Set(opts.channels.map((c) => c.replace(/^#/, ''))) : undefined;
+    for (const c of conversations) {
+        if (wanted && !wanted.has(c.name) && !wanted.has(c.id))
+            continue;
+        const folder = join(root, c.folder);
+        if (!(await stat(folder).catch(() => undefined))?.isDirectory())
+            continue;
+        const days = (await readdir(folder)).filter((f) => /^\d{4}-\d{2}-\d{2}\.json$/.test(f)).sort();
+        for (const day of days) {
+            const messages = (await readJson(join(folder, day))) ?? [];
+            if (!Array.isArray(messages))
+                continue;
+            for (const m of messages) {
+                if (!m || typeof m !== 'object' || !m.ts || (m.type && m.type !== 'message') || m.hidden)
+                    continue;
+                if (m.subtype && skip.has(m.subtype))
+                    continue;
+                const bot = m.subtype === 'bot_message' || (!!m.bot_id && !m.user);
+                if (bot && opts.skipBots)
+                    continue;
+                const extras = [
+                    ...(m.files ?? []).map((f) => f?.title || f?.name).filter((t) => !!t?.trim()).map((t) => `File: ${t}`),
+                    ...(m.attachments ?? []).map((a) => {
+                        const t = a?.title || a?.fallback;
+                        if (!t?.trim())
+                            return '';
+                        const link = a.title_link || a.from_url;
+                        return `Attachment: ${slackText(t, users, channelNames)}${link && !t.includes(link) ? ` (${link})` : ''}`;
+                    }).filter(Boolean),
+                ];
+                const said = slackText(m.text ?? '', users, channelNames).trim();
+                const text = [said, ...extras].filter(Boolean).join('\n');
+                if (!text)
+                    continue;
+                const author = (m.user && users.get(m.user)) || (m.user_profile && personName(m.user_profile)) || m.bot_profile?.name ||
+                    m.username || m.user || undefined;
+                const ms = Math.round(Number(m.ts) * 1000);
+                yield record({
+                    id: `${c.id}:${m.ts}`, text, kind: 'message', source, author, collection: opts.collection ?? c.name,
+                    thread: `${c.name}:${m.thread_ts ?? m.ts}`, at: Number.isFinite(ms) ? ms : undefined,
+                    url: opts.workspaceUrl ? `${opts.workspaceUrl.replace(/\/+$/, '')}/archives/${c.id}/p${m.ts.replace('.', '')}` : undefined,
+                    metadata: { channel: c.name, ts: m.ts, threadTs: m.thread_ts, subtype: m.subtype, replies: m.reply_count },
+                });
+            }
+        }
+    }
+}

package/dist/importers/text.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Small text tools the importers share: HTML reduced to text, character entities, bytes in any charset, long texts cut
+ * into parts. No dependency; any language and script is kept as written.
+ */
+import type { MemoryRecord } from '../records.js';
+/** `&`, `é`, `😀`, `é` and the other common names, decoded; unknown names are left as written. */
+export declare function decodeEntities(s: string): string;
+/** The `<title>` of an HTML page, else its first `<h1>`, as text; undefined when it has neither. */
+export declare function htmlTitle(html: string): string | undefined;
+/**
+ * HTML as plain text: scripts, styles, comments and the head dropped, blocks and line breaks kept as lines, list items
+ * as `- `, tags removed, entities decoded, `<pre>` kept as written. `dropQuotes` also drops `<blockquote>`s.
+ */
+export declare function htmlToText(html: string, opts?: {
+    dropQuotes?: boolean;
+}): string;
+/** Lines trimmed at the end, spaces and tabs inside a line kept to one, at most one empty line in a row. */
+export declare function tidy(s: string): string;
+/** Bytes as text in the charset named (any the platform knows); unnamed, ASCII or unknown: UTF-8, else Windows-1252. */
+export declare function decodeBytes(bytes: Uint8Array, charset?: string): string;
+/**
+ * A long text in parts of at most `maxChars`, each cut at a paragraph, else a line, else a sentence's end (in any
+ * script), else a space; never inside a character.
+ */
+export declare function splitText(text: string, maxChars: number): string[];
+/** A record whose fields, and metadata values, may be left undefined. */
+export type RecordDraft = Omit<MemoryRecord, 'metadata'> & {
+    metadata?: Record<string, string | number | boolean | null | undefined>;
+};
+/** A record without its empty fields. */
+export declare function record(r: RecordDraft): MemoryRecord;