@postalsys/email-content 0.0.1

package/dist/corpus/stats.js

@@ -0,0 +1,240 @@
+"use strict";
+/**
+ * Aggregate corpus statistics.
+ *
+ * Everything here is a count, a ratio or a percentile. No subject, address or body text is read,
+ * and the only free text that reaches the report is a MIME structure signature and detector ids,
+ * both of which are drawn from a closed vocabulary. That is what makes the report safe to paste
+ * into a document or an issue.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeStats = computeStats;
+exports.renderReport = renderReport;
+function toDistribution(rows, total, limit = 25) {
+    return rows
+        .filter(row => row.count > 0)
+        .sort((a, b) => b.count - a.count)
+        .slice(0, limit)
+        .map(row => ({
+        label: row.label === null || row.label === '' ? '(none)' : String(row.label),
+        count: row.count,
+        share: total ? row.count / total : 0
+    }));
+}
+/**
+ * Expand a comma separated column into per token counts.
+ *
+ * Grouping in SQL first keeps this cheap: a few thousand distinct combinations are fetched instead
+ * of one row per message.
+ */
+function expandMultiValue(store, column, total, limit = 30) {
+    const rows = store.query(`SELECT ${column} AS value, COUNT(*) AS count FROM messages GROUP BY ${column}`);
+    const counts = new Map();
+    for (const row of rows) {
+        const raw = row.value || '';
+        if (!raw) {
+            counts.set('(none)', (counts.get('(none)') || 0) + row.count);
+            continue;
+        }
+        for (const token of raw.split(',')) {
+            const key = token.trim();
+            if (key) {
+                counts.set(key, (counts.get(key) || 0) + row.count);
+            }
+        }
+    }
+    return toDistribution([...counts].map(([label, count]) => ({ label, count })), total, limit);
+}
+function groupCounts(store, column, table = 'messages') {
+    return store.query(`SELECT ${column} AS label, COUNT(*) AS count FROM ${table} GROUP BY ${column}`);
+}
+function percentile(store, fraction, total) {
+    if (!total) {
+        return 0;
+    }
+    const offset = Math.min(total - 1, Math.max(0, Math.floor(total * fraction)));
+    const row = store.queryOne('SELECT size FROM files ORDER BY size LIMIT 1 OFFSET ?', offset);
+    return row?.size ?? 0;
+}
+function computeStats(store) {
+    const fileTotal = store.queryOne('SELECT COUNT(*) AS count FROM files')?.count ?? 0;
+    const statusRows = groupCounts(store, 'status', 'files');
+    const statusOf = (status) => statusRows.find(row => row.label === status)?.count ?? 0;
+    const messageTotal = store.queryOne('SELECT COUNT(*) AS count FROM messages')?.count ?? 0;
+    const withMessageId = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE message_id IS NOT NULL AND message_id != ''")?.count ?? 0;
+    const withInReplyTo = store.queryOne('SELECT COUNT(*) AS count FROM messages WHERE in_reply_to IS NOT NULL')?.count ?? 0;
+    const parentResolved = store.queryOne('SELECT COUNT(*) AS count FROM messages m WHERE m.in_reply_to IS NOT NULL AND EXISTS (SELECT 1 FROM messages p WHERE p.message_id = m.in_reply_to)')?.count ?? 0;
+    const withReferences = store.queryOne('SELECT COUNT(*) AS count FROM messages WHERE references_count > 0')?.count ?? 0;
+    const ancestorResolved = store.queryOne('SELECT COUNT(*) AS count FROM messages m WHERE m.references_root IS NOT NULL AND EXISTS (SELECT 1 FROM messages p WHERE p.message_id = m.references_root)')?.count ?? 0;
+    const humanTotal = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE traffic = 'human'")?.count ?? 0;
+    const humanReplies = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE traffic = 'human' AND is_reply = 1")?.count ?? 0;
+    const humanRepliesResolved = store.queryOne("SELECT COUNT(*) AS count FROM messages m WHERE m.traffic = 'human' AND m.in_reply_to IS NOT NULL AND EXISTS (SELECT 1 FROM messages p WHERE p.message_id = m.in_reply_to)")?.count ?? 0;
+    const humanBoth = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE traffic = 'human' AND body_kind = 'both'")?.count ?? 0;
+    const humanDisagree = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE traffic = 'human' AND quote_marking_disagrees = 1")?.count ?? 0;
+    const distinctSha = store.queryOne('SELECT COUNT(DISTINCT sha256) AS count FROM files WHERE sha256 IS NOT NULL')?.count ?? 0;
+    const inDupGroups = store.queryOne('SELECT COALESCE(SUM(c), 0) AS count FROM (SELECT COUNT(*) AS c FROM files WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING c > 1)')?.count ?? 0;
+    const distinctSimhash = store.queryOne("SELECT COUNT(DISTINCT body_simhash) AS count FROM messages WHERE body_simhash IS NOT NULL AND body_simhash != '0000000000000000'")?.count ?? 0;
+    const inNearDupGroups = store.queryOne("SELECT COALESCE(SUM(c), 0) AS count FROM (SELECT COUNT(*) AS c FROM messages WHERE body_simhash IS NOT NULL AND body_simhash != '0000000000000000' GROUP BY body_simhash HAVING c > 1)")?.count ?? 0;
+    const bothRepresentations = store.queryOne("SELECT COUNT(*) AS count FROM messages WHERE body_kind = 'both'")?.count ?? 0;
+    const disagree = store.queryOne('SELECT COUNT(*) AS count FROM messages WHERE quote_marking_disagrees = 1')?.count ?? 0;
+    const totalBytes = store.queryOne('SELECT COALESCE(SUM(size), 0) AS total FROM files')?.total ?? 0;
+    return {
+        generatedAt: new Date().toISOString(),
+        files: {
+            total: fileTotal,
+            ok: statusOf('ok'),
+            failed: statusOf('failed'),
+            oversize: statusOf('oversize'),
+            timeout: statusOf('timeout'),
+            skipped: statusOf('skipped')
+        },
+        errors: toDistribution(groupCounts(store, 'error_code', 'files').filter(row => row.label), fileTotal),
+        sizeBytes: {
+            p10: percentile(store, 0.1, fileTotal),
+            p50: percentile(store, 0.5, fileTotal),
+            p90: percentile(store, 0.9, fileTotal),
+            p99: percentile(store, 0.99, fileTotal),
+            max: store.queryOne('SELECT MAX(size) AS size FROM files')?.size ?? 0,
+            total: totalBytes
+        },
+        parseWarnings: expandMultiValue(store, 'warnings', messageTotal),
+        bodyKind: toDistribution(groupCounts(store, 'body_kind'), messageTotal),
+        structures: toDistribution(groupCounts(store, 'structure'), messageTotal, 25),
+        charsets: expandMultiValue(store, 'charsets', messageTotal, 20),
+        encodings: expandMultiValue(store, 'encodings', messageTotal, 12),
+        traffic: toDistribution(groupCounts(store, 'traffic'), messageTotal),
+        clients: toDistribution(groupCounts(store, 'client'), messageTotal, 25),
+        clientSources: toDistribution(groupCounts(store, 'client_source'), messageTotal),
+        subjectPrefix: toDistribution(groupCounts(store, 'subject_prefix'), messageTotal),
+        replyStyle: toDistribution(groupCounts(store, 'reply_style'), messageTotal),
+        quoteDepth: toDistribution(groupCounts(store, 'quote_depth_max'), messageTotal, 10),
+        markers: expandMultiValue(store, 'markers', messageTotal, 30),
+        htmlHints: expandMultiValue(store, 'html_hints', messageTotal, 25),
+        languages: toDistribution(groupCounts(store, 'lang'), messageTotal, 15),
+        duplicates: {
+            distinctSha256: distinctSha,
+            filesInDuplicateGroups: inDupGroups,
+            distinctSimhash,
+            filesInNearDuplicateGroups: inNearDupGroups
+        },
+        threading: {
+            withMessageId,
+            withInReplyTo,
+            parentResolvedInCorpus: parentResolved,
+            withReferences,
+            ancestorResolvedInCorpus: ancestorResolved
+        },
+        humanSlice: {
+            total: humanTotal,
+            replies: humanReplies,
+            repliesWithParentInCorpus: humanRepliesResolved,
+            bothRepresentations: humanBoth,
+            quoteMarkingDisagrees: humanDisagree
+        },
+        contentAgreement: {
+            messagesWithBoth: bothRepresentations,
+            quoteMarkingDisagrees: disagree
+        }
+    };
+}
+function pct(value, total) {
+    if (!total) {
+        return '0.0%';
+    }
+    return `${((100 * value) / total).toFixed(1)}%`;
+}
+function bytes(value) {
+    if (value >= 1024 * 1024 * 1024) {
+        return `${(value / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+    }
+    if (value >= 1024 * 1024) {
+        return `${(value / (1024 * 1024)).toFixed(1)} MB`;
+    }
+    if (value >= 1024) {
+        return `${(value / 1024).toFixed(1)} KB`;
+    }
+    return `${value} B`;
+}
+function table(title, rows) {
+    if (!rows.length) {
+        return `### ${title}\n\nNo data.\n`;
+    }
+    const lines = ['### ' + title, '', '| value | count | share |', '| --- | ---: | ---: |'];
+    for (const row of rows) {
+        lines.push(`| \`${row.label.replace(/\|/g, '\\|')}\` | ${row.count.toLocaleString('en-US')} | ${(row.share * 100).toFixed(1)}% |`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/** Render the statistics as a Markdown report. Safe to share: contains no message content. */
+function renderReport(stats, root) {
+    const files = stats.files;
+    const messages = stats.files.ok;
+    const out = [];
+    out.push('# Corpus report', '');
+    out.push(`Generated ${stats.generatedAt} from \`${root}\`.`, '');
+    out.push('This report contains aggregate statistics only. No subject lines, addresses or body text are', 'stored or displayed. Detector names come from a closed vocabulary defined in', '`src/detect/patterns/`.', '');
+    out.push('## Files', '');
+    out.push('| metric | value |', '| --- | ---: |');
+    out.push(`| files scanned | ${files.total.toLocaleString('en-US')} |`);
+    out.push(`| parsed successfully | ${files.ok.toLocaleString('en-US')} (${pct(files.ok, files.total)}) |`);
+    out.push(`| parse failures | ${files.failed.toLocaleString('en-US')} (${pct(files.failed, files.total)}) |`);
+    out.push(`| oversize, not parsed | ${files.oversize.toLocaleString('en-US')} |`);
+    out.push(`| timed out | ${files.timeout.toLocaleString('en-US')} |`);
+    out.push(`| total bytes | ${bytes(stats.sizeBytes.total)} |`);
+    out.push(`| size p10 / p50 / p90 / p99 / max | ${bytes(stats.sizeBytes.p10)} / ${bytes(stats.sizeBytes.p50)} / ${bytes(stats.sizeBytes.p90)} / ${bytes(stats.sizeBytes.p99)} / ${bytes(stats.sizeBytes.max)} |`);
+    out.push('');
+    if (stats.errors.length) {
+        out.push(table('Failure reasons', stats.errors));
+    }
+    out.push('## Threading', '');
+    out.push('| metric | value | share |', '| --- | ---: | ---: |');
+    out.push(`| messages with Message-ID | ${stats.threading.withMessageId.toLocaleString('en-US')} | ${pct(stats.threading.withMessageId, messages)} |`);
+    out.push(`| messages with In-Reply-To | ${stats.threading.withInReplyTo.toLocaleString('en-US')} | ${pct(stats.threading.withInReplyTo, messages)} |`);
+    out.push(`| **parent present in corpus** | ${stats.threading.parentResolvedInCorpus.toLocaleString('en-US')} | ${pct(stats.threading.parentResolvedInCorpus, stats.threading.withInReplyTo)} of replies |`);
+    out.push(`| messages with References | ${stats.threading.withReferences.toLocaleString('en-US')} | ${pct(stats.threading.withReferences, messages)} |`);
+    out.push(`| thread root present in corpus | ${stats.threading.ancestorResolvedInCorpus.toLocaleString('en-US')} | ${pct(stats.threading.ancestorResolvedInCorpus, stats.threading.withReferences)} of those |`);
+    out.push('');
+    out.push('Parent resolution is what makes distant supervision possible: when the parent message is in', 'the corpus, any text the extractor hides can be checked against text that demonstrably', 'existed before, without a human labelling it.', '');
+    out.push('## Human conversation slice', '');
+    out.push('| metric | value | share |', '| --- | ---: | ---: |');
+    out.push(`| classified human | ${stats.humanSlice.total.toLocaleString('en-US')} | ${pct(stats.humanSlice.total, messages)} of all |`);
+    out.push(`| human replies | ${stats.humanSlice.replies.toLocaleString('en-US')} | ${pct(stats.humanSlice.replies, stats.humanSlice.total)} of human |`);
+    out.push(`| human replies with parent in corpus | ${stats.humanSlice.repliesWithParentInCorpus.toLocaleString('en-US')} | ${pct(stats.humanSlice.repliesWithParentInCorpus, stats.humanSlice.replies)} of human replies |`);
+    out.push(`| human, both plain and HTML | ${stats.humanSlice.bothRepresentations.toLocaleString('en-US')} | ${pct(stats.humanSlice.bothRepresentations, stats.humanSlice.total)} |`);
+    out.push(`| human, plain and HTML disagree on quoting | ${stats.humanSlice.quoteMarkingDisagrees.toLocaleString('en-US')} | ${pct(stats.humanSlice.quoteMarkingDisagrees, stats.humanSlice.bothRepresentations)} of those |`);
+    out.push('');
+    out.push('## Duplicates', '');
+    out.push('| metric | value |', '| --- | ---: |');
+    out.push(`| distinct file hashes | ${stats.duplicates.distinctSha256.toLocaleString('en-US')} of ${files.total.toLocaleString('en-US')} |`);
+    out.push(`| files in exact duplicate groups | ${stats.duplicates.filesInDuplicateGroups.toLocaleString('en-US')} |`);
+    out.push(`| distinct body SimHashes | ${stats.duplicates.distinctSimhash.toLocaleString('en-US')} |`);
+    out.push(`| files sharing a body SimHash | ${stats.duplicates.filesInNearDuplicateGroups.toLocaleString('en-US')} |`);
+    out.push('');
+    out.push('SimHash collisions are a lower bound on near duplicates: only identical hashes are counted here, not small Hamming distances.', '');
+    out.push('## Structure', '');
+    out.push(table('Body composition', stats.bodyKind));
+    out.push(table('MIME structures', stats.structures));
+    out.push(table('Charsets', stats.charsets));
+    out.push(table('Transfer encodings', stats.encodings));
+    out.push(table('Parse warnings', stats.parseWarnings));
+    out.push('## Traffic and clients', '');
+    out.push(table('Traffic class', stats.traffic));
+    out.push(table('Composing client', stats.clients));
+    out.push(table('Client evidence source', stats.clientSources));
+    out.push('## Reply structure', '');
+    out.push(table('Subject prefix', stats.subjectPrefix));
+    out.push(table('Plain text reply style', stats.replyStyle));
+    out.push(table('Maximum quote depth', stats.quoteDepth));
+    out.push(table('Line level markers detected', stats.markers));
+    out.push(table('HTML structure hints', stats.htmlHints));
+    out.push('## Language', '');
+    out.push(table('Estimated language', stats.languages));
+    out.push('## Plain text and HTML agreement', '');
+    out.push(`Messages carrying both representations: ${stats.contentAgreement.messagesWithBoth.toLocaleString('en-US')}.`, '');
+    out.push(`Quote marking disagrees between the two in ${stats.contentAgreement.quoteMarkingDisagrees.toLocaleString('en-US')} of them ` +
+        `(${pct(stats.contentAgreement.quoteMarkingDisagrees, stats.contentAgreement.messagesWithBoth)}).`, '');
+    out.push('Disagreement is the norm rather than the exception, and it is almost always the plain part', 'that has lost the quote markers. That is why HTML is the primary representation when both', 'are present.', '');
+    return out.join('\n');
+}
+//# sourceMappingURL=stats.js.map

package/dist/corpus/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/corpus/stats.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AA0HH,oCA8GC;AAmCD,oCA+GC;AAtTD,SAAS,cAAc,CAAC,IAAyB,EAAE,KAAa,EAAE,KAAK,GAAG,EAAE;IACxE,OAAO,IAAI;SACN,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACT,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;QAC5E,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;KACvC,CAAC,CAAC,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,KAAkB,EAAE,MAAc,EAAE,KAAa,EAAE,KAAK,GAAG,EAAE;IACnF,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAA0C,UAAU,MAAM,uDAAuD,MAAM,EAAE,CAAC,CAAC;IACnJ,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9D,SAAS;QACb,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YACzB,IAAI,GAAG,EAAE,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YACxD,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,cAAc,CACjB,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,EACvD,KAAK,EACL,KAAK,CACR,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,KAAkB,EAAE,MAAc,EAAE,KAAK,GAAG,UAAU;IACvE,OAAO,KAAK,CAAC,KAAK,CAAW,UAAU,MAAM,qCAAqC,KAAK,aAAa,MAAM,EAAE,CAAC,CAAC;AAClH,CAAC;AAED,SAAS,UAAU,CAAC,KAAkB,EAAE,QAAgB,EAAE,KAAa;IACnE,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,OAAO,CAAC,CAAC;IACb,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAmB,uDAAuD,EAAE,MAAM,CAAC,CAAC;IAC9G,OAAO,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAgB,YAAY,CAAC,KAAkB;IAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAoB,qCAAqC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACvG,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAE9F,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAoB,wCAAwC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAE7G,MAAM,aAAa,GACf,KAAK,CAAC,QAAQ,CAAoB,0FAA0F,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAC9I,MAAM,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAoB,sEAAsE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAC5I,MAAM,cAAc,GAChB,KAAK,CAAC,QAAQ,CACV,mJAAmJ,CACtJ,EAAE,KAAK,IAAI,CAAC,CAAC;IAClB,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAoB,mEAAmE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAC1I,MAAM,gBAAgB,GAClB,KAAK,CAAC,QAAQ,CACV,2JAA2J,CAC9J,EAAE,KAAK,IAAI,CAAC,CAAC;IAElB,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAoB,gEAAgE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACnI,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAoB,iFAAiF,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACtJ,MAAM,oBAAoB,GACtB,KAAK,CAAC,QAAQ,CACV,2KAA2K,CAC9K,EAAE,KAAK,IAAI,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAoB,uFAAuF,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACzJ,MAAM,aAAa,GACf,KAAK,CAAC,QAAQ,CAAoB,gGAAgG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAEpJ,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAoB,4EAA4E,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAChJ,MAAM,WAAW,GACb,KAAK,CAAC,QAAQ,CACV,kIAAkI,CACrI,EAAE,KAAK,IAAI,CAAC,CAAC;IAClB,MAAM,eAAe,GACjB,KAAK,CAAC,QAAQ,CACV,kIAAkI,CACrI,EAAE,KAAK,IAAI,CAAC,CAAC;IAClB,MAAM,eAAe,GACjB,KAAK,CAAC,QAAQ,CACV,wLAAwL,CAC3L,EAAE,KAAK,IAAI,CAAC,CAAC;IAElB,MAAM,mBAAmB,GAAG,KAAK,CAAC,QAAQ,CAAoB,iEAAiE,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAC7I,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAoB,0EAA0E,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAE3I,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAoB,mDAAmD,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAEtH,OAAO;QACH,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,KAAK,EAAE;YACH,KAAK,EAAE,SAAS;YAChB,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC;YAClB,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC;YAC1B,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;YAC9B,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC;YAC5B,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC;SAC/B;QACD,MAAM,EAAE,cAAc,CAClB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAClE,SAAS,CACZ;QACD,SAAS,EAAE;YACP,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;YACtC,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;YACtC,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;YACtC,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC;YACvC,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAmB,qCAAqC,CAAC,EAAE,IAAI,IAAI,CAAC;YACvF,KAAK,EAAE,UAAU;SACpB;QACD,aAAa,EAAE,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC;QAChE,QAAQ,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,YAAY,CAAC;QACvE,UAAU,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;QAC7E,QAAQ,EAAE,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC;QAC/D,SAAS,EAAE,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,EAAE,CAAC;QACjE,OAAO,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,YAAY,CAAC;QACpE,OAAO,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;QACvE,aAAa,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,eAAe,CAAC,EAAE,YAAY,CAAC;QAChF,aAAa,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,CAAC,EAAE,YAAY,CAAC;QACjF,UAAU,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC;QAC3E,UAAU,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;QACnF,OAAO,EAAE,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,CAAC;QAC7D,SAAS,EAAE,gBAAgB,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;QAClE,SAAS,EAAE,cAAc,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;QACvE,UAAU,EAAE;YACR,cAAc,EAAE,WAAW;YAC3B,sBAAsB,EAAE,WAAW;YACnC,eAAe;YACf,0BAA0B,EAAE,eAAe;SAC9C;QACD,SAAS,EAAE;YACP,aAAa;YACb,aAAa;YACb,sBAAsB,EAAE,cAAc;YACtC,cAAc;YACd,wBAAwB,EAAE,gBAAgB;SAC7C;QACD,UAAU,EAAE;YACR,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE,YAAY;YACrB,yBAAyB,EAAE,oBAAoB;YAC/C,mBAAmB,EAAE,SAAS;YAC9B,qBAAqB,EAAE,aAAa;SACvC;QACD,gBAAgB,EAAE;YACd,gBAAgB,EAAE,mBAAmB;YACrC,qBAAqB,EAAE,QAAQ;SAClC;KACJ,CAAC;AACN,CAAC;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACrC,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpD,CAAC;AAED,SAAS,KAAK,CAAC,KAAa;IACxB,IAAI,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;QAC9B,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7D,CAAC;IACD,IAAI,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAChB,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,GAAG,KAAK,IAAI,CAAC;AACxB,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,IAA6B;IACvD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,OAAO,KAAK,gBAAgB,CAAC;IACxC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,EAAE,EAAE,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;IACzF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACvI,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,8FAA8F;AAC9F,SAAgB,YAAY,CAAC,KAAkB,EAAE,IAAY;IACzD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;IAChC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAChC,GAAG,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,WAAW,WAAW,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CACJ,8FAA8F,EAC9F,8EAA8E,EAC9E,yBAAyB,EACzB,EAAE,CACL,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACzB,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;IACjD,GAAG,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACvE,GAAG,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1G,GAAG,CAAC,IAAI,CAAC,sBAAsB,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7G,GAAG,CAAC,IAAI,CAAC,4BAA4B,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjF,GAAG,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,GAAG,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9D,GAAG,CAAC,IAAI,CACJ,wCAAwC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CACzM,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC7B,GAAG,CAAC,IAAI,CAAC,4BAA4B,EAAE,uBAAuB,CAAC,CAAC;IAChE,GAAG,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACtJ,GAAG,CAAC,IAAI,CAAC,iCAAiC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvJ,GAAG,CAAC,IAAI,CACJ,oCAAoC,KAAK,CAAC,SAAS,CAAC,sBAAsB,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,sBAAsB,EAAE,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,eAAe,CACpM,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxJ,GAAG,CAAC,IAAI,CACJ,qCAAqC,KAAK,CAAC,SAAS,CAAC,wBAAwB,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,wBAAwB,EAAE,KAAK,CAAC,SAAS,CAAC,cAAc,CAAC,aAAa,CACxM,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CACJ,6FAA6F,EAC7F,wFAAwF,EACxF,+CAA+C,EAC/C,EAAE,CACL,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;IAC5C,GAAG,CAAC,IAAI,CAAC,4BAA4B,EAAE,uBAAuB,CAAC,CAAC;IAChE,GAAG,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvI,GAAG,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxJ,GAAG,CAAC,IAAI,CACJ,2CAA2C,KAAK,CAAC,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,yBAAyB,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,qBAAqB,CACpN,CAAC;IACF,GAAG,CAAC,IAAI,CACJ,kCAAkC,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,mBAAmB,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAC5K,CAAC;IACF,GAAG,CAAC,IAAI,CACJ,iDAAiD,KAAK,CAAC,UAAU,CAAC,qBAAqB,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,qBAAqB,EAAE,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,aAAa,CACtN,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAC;IACjD,GAAG,CAAC,IAAI,CAAC,4BAA4B,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5I,GAAG,CAAC,IAAI,CAAC,uCAAuC,KAAK,CAAC,UAAU,CAAC,sBAAsB,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrH,GAAG,CAAC,IAAI,CAAC,+BAA+B,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtG,GAAG,CAAC,IAAI,CAAC,oCAAoC,KAAK,CAAC,UAAU,CAAC,0BAA0B,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtH,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CAAC,+HAA+H,EAAE,EAAE,CAAC,CAAC;IAE9I,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC7B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACrD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;IAEvD,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;IACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;IAE/D,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACnC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;IACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5D,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACzD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9D,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAEzD,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAEvD,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAC;IACjD,GAAG,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5H,GAAG,CAAC,IAAI,CACJ,8CAA8C,KAAK,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,cAAc,CAAC,OAAO,CAAC,WAAW;QACzH,IAAI,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,KAAK,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,EACtG,EAAE,CACL,CAAC;IACF,GAAG,CAAC,IAAI,CACJ,4FAA4F,EAC5F,2FAA2F,EAC3F,cAAc,EACd,EAAE,CACL,CAAC;IAEF,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/corpus/store.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Corpus metadata store.
+ *
+ * node:sqlite is used rather than better-sqlite3 or DuckDB: it is built into Node 24, so there is
+ * no native build step and no lockfile that can stop working in two years. The corpus is a few
+ * hundred thousand rows of small metadata, which is well within its comfortable range.
+ *
+ * The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
+ * never copied. See docs/threat-model.md for what the derived data can still reveal.
+ */
+import { type MessageRecord } from './analyze';
+export type FileStatus = 'ok' | 'failed' | 'skipped' | 'oversize' | 'timeout';
+export interface FileRecord {
+    path: string;
+    size: number;
+    mtime: number;
+    sha256: string | null;
+    status: FileStatus;
+    errorCode: string | null;
+    errorDetail: string | null;
+    durationMs: number | null;
+}
+export interface ScanRow {
+    file: FileRecord;
+    message: MessageRecord | null;
+}
+export interface RunOptions {
+    root: string;
+    [key: string]: unknown;
+}
+export declare class CorpusStore {
+    private readonly db;
+    private readonly insertFileStmt;
+    private readonly insertMessageStmt;
+    private readonly seenStmt;
+    /** Per database salt for privacy preserving hashes. Created on first open. */
+    readonly salt: string;
+    constructor(file: string);
+    private migrate;
+    private readOrCreateSalt;
+    getMeta(key: string): string | null;
+    setMeta(key: string, value: string): void;
+    startRun(options: RunOptions): number;
+    finishRun(runId: number, status: 'completed' | 'interrupted' | 'failed', counts: {
+        seen: number;
+        processed: number;
+        failed: number;
+        skipped: number;
+    }): void;
+    /**
+     * True when this exact file has already been processed by the current processing version.
+     * Resume relies on size and mtime rather than a hash, because hashing every file would mean
+     * reading the whole corpus again just to decide what to skip.
+     */
+    isProcessed(relPath: string, size: number, mtime: number): boolean;
+    /** Write a batch inside a single transaction. */
+    writeBatch(rows: readonly ScanRow[], runId: number): void;
+    /** Execute a statement. Returns the driver result, including lastInsertRowid. */
+    run(sql: string, ...params: (string | number | null)[]): import("node:sqlite").StatementResultingChanges;
+    /** Run a function inside a transaction, rolling back on any throw. */
+    transaction(fn: () => void): void;
+    /** Run an arbitrary read only query. Used by the stats and sampling commands. */
+    query<T = Record<string, unknown>>(sql: string, ...params: (string | number | null)[]): T[];
+    queryOne<T = Record<string, unknown>>(sql: string, ...params: (string | number | null)[]): T | null;
+    close(): void;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/corpus/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,MAAM,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAe;IAClC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAE1B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM;IAuDxB,OAAO,CAAC,OAAO;IAkBf,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKnC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIzC,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;IASrC,SAAS,CACL,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,QAAQ,EAC9C,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAC7E,IAAI;IAMP;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAQlE,iDAAiD;IACjD,UAAU,CAAC,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IA6EzD,iFAAiF;IACjF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;IAItD,sEAAsE;IACtE,WAAW,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI;IAWjC,iFAAiF;IACjF,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE;IAI3F,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI;IAInG,KAAK,IAAI,IAAI;CAQhB"}

package/dist/corpus/store.js

@@ -0,0 +1,196 @@
+"use strict";
+/**
+ * Corpus metadata store.
+ *
+ * node:sqlite is used rather than better-sqlite3 or DuckDB: it is built into Node 24, so there is
+ * no native build step and no lockfile that can stop working in two years. The corpus is a few
+ * hundred thousand rows of small metadata, which is well within its comfortable range.
+ *
+ * The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
+ * never copied. See docs/threat-model.md for what the derived data can still reveal.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CorpusStore = void 0;
+const node_sqlite_1 = require("node:sqlite");
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const types_1 = require("../types");
+const hash_1 = require("../util/hash");
+const migrations_1 = require("./migrations");
+class CorpusStore {
+    db;
+    insertFileStmt;
+    insertMessageStmt;
+    seenStmt;
+    /** Per database salt for privacy preserving hashes. Created on first open. */
+    salt;
+    constructor(file) {
+        const dir = node_path_1.default.dirname(node_path_1.default.resolve(file));
+        node_fs_1.default.mkdirSync(dir, { recursive: true });
+        this.db = new node_sqlite_1.DatabaseSync(file);
+        this.db.exec('PRAGMA journal_mode = WAL');
+        this.db.exec('PRAGMA synchronous = NORMAL');
+        this.db.exec('PRAGMA foreign_keys = ON');
+        this.migrate();
+        this.salt = this.readOrCreateSalt();
+        this.insertFileStmt = this.db.prepare(`
+            INSERT INTO files (path, size, mtime, sha256, status, error_code, error_detail, duration_ms, scanned_at, run_id, processing_version)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(path) DO UPDATE SET
+                size = excluded.size, mtime = excluded.mtime, sha256 = excluded.sha256,
+                status = excluded.status, error_code = excluded.error_code, error_detail = excluded.error_detail,
+                duration_ms = excluded.duration_ms, scanned_at = excluded.scanned_at, run_id = excluded.run_id,
+                processing_version = excluded.processing_version
+        `);
+        this.insertMessageStmt = this.db.prepare(`
+            INSERT INTO messages (
+                path, message_id, in_reply_to, references_count, references_root, date_ms,
+                from_domain, from_hash, subject_hash, subject_prefix, subject_length,
+                structure, body_kind, charsets, encodings, part_count, attachment_count, has_rfc822,
+                text_chars, html_chars, flowed, client, client_source, traffic, traffic_signals,
+                is_reply, is_forward, quote_depth_max, quoted_ratio, reply_style, markers, html_hints,
+                blockquote_count, table_count, has_image, suspect_malformed_html, lang, lang_score,
+                body_simhash, warnings, quote_marking_disagrees
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(path) DO UPDATE SET
+                message_id = excluded.message_id, in_reply_to = excluded.in_reply_to,
+                references_count = excluded.references_count, references_root = excluded.references_root,
+                date_ms = excluded.date_ms, from_domain = excluded.from_domain, from_hash = excluded.from_hash,
+                subject_hash = excluded.subject_hash, subject_prefix = excluded.subject_prefix,
+                subject_length = excluded.subject_length, structure = excluded.structure,
+                body_kind = excluded.body_kind, charsets = excluded.charsets, encodings = excluded.encodings,
+                part_count = excluded.part_count, attachment_count = excluded.attachment_count,
+                has_rfc822 = excluded.has_rfc822, text_chars = excluded.text_chars, html_chars = excluded.html_chars,
+                flowed = excluded.flowed, client = excluded.client, client_source = excluded.client_source,
+                traffic = excluded.traffic, traffic_signals = excluded.traffic_signals, is_reply = excluded.is_reply,
+                is_forward = excluded.is_forward, quote_depth_max = excluded.quote_depth_max,
+                quoted_ratio = excluded.quoted_ratio, reply_style = excluded.reply_style, markers = excluded.markers,
+                html_hints = excluded.html_hints, blockquote_count = excluded.blockquote_count,
+                table_count = excluded.table_count, has_image = excluded.has_image,
+                suspect_malformed_html = excluded.suspect_malformed_html, lang = excluded.lang,
+                lang_score = excluded.lang_score, body_simhash = excluded.body_simhash,
+                warnings = excluded.warnings, quote_marking_disagrees = excluded.quote_marking_disagrees
+        `);
+        this.seenStmt = this.db.prepare('SELECT size, mtime, processing_version FROM files WHERE path = ?');
+    }
+    migrate() {
+        const current = Number(this.db.prepare('PRAGMA user_version').get()?.user_version ?? 0);
+        for (const migration of migrations_1.MIGRATIONS) {
+            if (migration.version <= current) {
+                continue;
+            }
+            this.db.exec('BEGIN');
+            try {
+                this.db.exec(migration.sql);
+                this.db.exec(`PRAGMA user_version = ${migration.version}`);
+                this.db.exec('COMMIT');
+            }
+            catch (err) {
+                this.db.exec('ROLLBACK');
+                throw new Error(`migration ${migration.version} (${migration.name}) failed`, { cause: err });
+            }
+        }
+    }
+    readOrCreateSalt() {
+        const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get('salt');
+        if (row?.value) {
+            return row.value;
+        }
+        const salt = (0, hash_1.newSalt)();
+        this.db.prepare('INSERT INTO meta (key, value) VALUES (?, ?)').run('salt', salt);
+        return salt;
+    }
+    getMeta(key) {
+        const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
+        return row?.value ?? null;
+    }
+    setMeta(key, value) {
+        this.db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
+    }
+    startRun(options) {
+        const stmt = this.db.prepare(`
+            INSERT INTO runs (root, started_at, status, processing_version, extractor_version, options_json)
+            VALUES (?, ?, 'running', ?, ?, ?)
+        `);
+        const result = stmt.run(options.root, Date.now(), types_1.PROCESSING_VERSION, types_1.EXTRACTOR_VERSION, JSON.stringify(options));
+        return Number(result.lastInsertRowid);
+    }
+    finishRun(runId, status, counts) {
+        this.db
+            .prepare('UPDATE runs SET finished_at = ?, status = ?, files_seen = ?, files_processed = ?, files_failed = ?, files_skipped = ? WHERE id = ?')
+            .run(Date.now(), status, counts.seen, counts.processed, counts.failed, counts.skipped, runId);
+    }
+    /**
+     * True when this exact file has already been processed by the current processing version.
+     * Resume relies on size and mtime rather than a hash, because hashing every file would mean
+     * reading the whole corpus again just to decide what to skip.
+     */
+    isProcessed(relPath, size, mtime) {
+        const row = this.seenStmt.get(relPath);
+        if (!row) {
+            return false;
+        }
+        return row.size === size && row.mtime === mtime && row.processing_version === types_1.PROCESSING_VERSION;
+    }
+    /** Write a batch inside a single transaction. */
+    writeBatch(rows, runId) {
+        if (!rows.length) {
+            return;
+        }
+        this.db.exec('BEGIN');
+        try {
+            const now = Date.now();
+            for (const row of rows) {
+                const file = row.file;
+                this.insertFileStmt.run(file.path, file.size, file.mtime, file.sha256, file.status, file.errorCode, file.errorDetail, file.durationMs, now, runId, types_1.PROCESSING_VERSION);
+                const record = row.message;
+                if (!record) {
+                    continue;
+                }
+                this.insertMessageStmt.run(file.path, record.messageId, record.inReplyTo, record.referencesCount, record.referencesRoot, record.dateMs, record.fromDomain, record.fromHash, record.subjectHash, record.subjectPrefix, record.subjectLength, record.structure, record.bodyKind, record.charsets, record.encodings, record.partCount, record.attachmentCount, record.hasRfc822 ? 1 : 0, record.textChars, record.htmlChars, record.flowed ? 1 : 0, record.client, record.clientSource, record.traffic, record.trafficSignals, record.isReply ? 1 : 0, record.isForward ? 1 : 0, record.quoteDepthMax, record.quotedRatio, record.replyStyle, record.markers, record.htmlHints, record.blockquoteCount, record.tableCount, record.hasImage ? 1 : 0, record.suspectMalformedHtml ? 1 : 0, record.lang, record.langScore, record.bodySimhash, record.warnings, record.quoteMarkingDisagrees ? 1 : 0);
+            }
+            this.db.exec('COMMIT');
+        }
+        catch (err) {
+            this.db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+    /** Execute a statement. Returns the driver result, including lastInsertRowid. */
+    run(sql, ...params) {
+        return this.db.prepare(sql).run(...params);
+    }
+    /** Run a function inside a transaction, rolling back on any throw. */
+    transaction(fn) {
+        this.db.exec('BEGIN');
+        try {
+            fn();
+            this.db.exec('COMMIT');
+        }
+        catch (err) {
+            this.db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+    /** Run an arbitrary read only query. Used by the stats and sampling commands. */
+    query(sql, ...params) {
+        return this.db.prepare(sql).all(...params);
+    }
+    queryOne(sql, ...params) {
+        return this.db.prepare(sql).get(...params) ?? null;
+    }
+    close() {
+        try {
+            this.db.exec('PRAGMA optimize');
+        }
+        catch {
+            // Best effort only. A failure here must not mask a successful scan.
+        }
+        this.db.close();
+    }
+}
+exports.CorpusStore = CorpusStore;
+//# sourceMappingURL=store.js.map

package/dist/corpus/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;AAEH,6CAA2C;AAC3C,sDAAyB;AACzB,0DAA6B;AAC7B,oCAAiE;AACjE,uCAAuC;AACvC,6CAA0C;AA0B1C,MAAa,WAAW;IACH,EAAE,CAAe;IACjB,cAAc,CAAC;IACf,iBAAiB,CAAC;IAClB,QAAQ,CAAC;IAE1B,8EAA8E;IACrE,IAAI,CAAS;IAEtB,YAAY,IAAY;QACpB,MAAM,GAAG,GAAG,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,iBAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,IAAI,CAAC,EAAE,GAAG,IAAI,0BAAY,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEpC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;SAQrC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA4BxC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kEAAkE,CAAC,CAAC;IACxG,CAAC;IAEO,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,CAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAgC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QACvH,KAAK,MAAM,SAAS,IAAI,uBAAU,EAAE,CAAC;YACjC,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC;gBAC/B,SAAS;YACb,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,IAAI,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACL,CAAC;IACL,CAAC;IAEO,gBAAgB;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAmC,CAAC;QAClH,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,KAAK,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,cAAO,GAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,GAAW;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAmC,CAAC;QAC/G,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mGAAmG,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzI,CAAC;IAED,QAAQ,CAAC,OAAmB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,0BAAkB,EAAE,yBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAClH,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC;IAED,SAAS,CACL,KAAa,EACb,MAA8C,EAC9C,MAA4E;QAE5E,IAAI,CAAC,EAAE;aACF,OAAO,CAAC,oIAAoI,CAAC;aAC7I,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACtG,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAa;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAA+E,CAAC;QACrH,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,kBAAkB,KAAK,0BAAkB,CAAC;IACrG,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,IAAwB,EAAE,KAAa;QAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;gBACtB,IAAI,CAAC,cAAc,CAAC,GAAG,CACnB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,UAAU,EACf,GAAG,EACH,KAAK,EACL,0BAAkB,CACrB,CAAC;gBACF,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACtB,IAAI,CAAC,IAAI,EACT,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvB,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnC,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvC,CAAC;YACN,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAW,EAAE,GAAG,MAAkC;QAClD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,sEAAsE;IACtE,WAAW,CAAC,EAAc;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,EAAE,EAAE,CAAC;YACL,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,KAAK,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACjF,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAQ,CAAC;IACtD,CAAC;IAED,QAAQ,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACpF,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAO,IAAI,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK;QACD,IAAI,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACL,oEAAoE;QACxE,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACJ;AArPD,kCAqPC"}

package/dist/corpus/thread-index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Thread reconstruction over a scanned corpus.
+ *
+ * Resolves a message's In-Reply-To and References to files that are actually present, so that
+ * Stage 5 has something to match against. Measured over the working corpus, 90.9 percent of replies
+ * resolve their direct parent this way.
+ *
+ * Ancestors are returned nearest first. The direct parent carries most of the value: it is what the
+ * sender's client quoted from. Older ancestors matter for deeply nested threads, where the reply
+ * contains several generations of history at once.
+ */
+import { type HistoryDocument } from '../match/history';
+import { type MessageHeaders } from '../mime/model';
+import { type CorpusStore } from './store';
+export interface ThreadResolver {
+    /** Load history documents for a message, nearest ancestor first. */
+    resolve(headers: MessageHeaders, maxAncestors?: number): Promise<HistoryDocument[]>;
+}
+/** How many ancestors to load by default. Each one costs a file read and a parse. */
+export declare const DEFAULT_MAX_ANCESTORS = 3;
+/**
+ * Build a resolver backed by a scanned corpus database.
+ *
+ * The lookup is a single indexed query per candidate Message-ID, so no in memory index of the whole
+ * corpus is needed. That matters: a Message-ID map over 320k messages is tens of megabytes that
+ * would otherwise be paid for on every process start.
+ */
+export declare function corpusThreadResolver(store: CorpusStore, root: string): ThreadResolver;
+/** Build a resolver from a folder of .eml files, for callers with no corpus database. */
+export declare function folderThreadResolver(folder: string): ThreadResolver;
+//# sourceMappingURL=thread-index.d.ts.map

package/dist/corpus/thread-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"thread-index.d.ts","sourceRoot":"","sources":["../../src/corpus/thread-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,EAAwB,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,WAAW,cAAc;IAC3B,oEAAoE;IACpE,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACvF;AAED,qFAAqF;AACrF,eAAO,MAAM,qBAAqB,IAAI,CAAC;AAEvC;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,CAqDrF;AAED,yFAAyF;AACzF,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAiEnE"}