@postalsys/email-content 0.0.1

package/dist/corpus/scanner.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Streaming corpus scanner.
+ *
+ * Requirements this design answers, in order of importance:
+ *   - one malformed message must never stop a run, so every failure becomes a row
+ *   - bounded memory, so files are streamed and results are flushed in batches
+ *   - resumable, so an interrupted run skips what it already recorded
+ *   - deterministic, so the same seed always produces the same sample
+ *   - no message content in any log line
+ *
+ * Backpressure is driven from here rather than inside the pool: the scanner keeps a bounded window
+ * of in flight tasks and awaits a completion before dispatching more. That keeps peak memory
+ * proportional to the window size rather than to the corpus size.
+ */
+export interface ScanOptions {
+    root: string;
+    dbPath: string;
+    /** Number of worker threads. Defaults to cores minus two, minimum one. */
+    concurrency?: number;
+    /** Deterministic sample size. Zero or undefined scans everything. */
+    sample?: number;
+    /** Seed for deterministic sampling. */
+    seed?: string;
+    /** Skip files already recorded by this processing version. Default true. */
+    resume?: boolean;
+    /** Files larger than this are recorded as oversize and not parsed. Default 32 MiB. */
+    maxFileBytes?: number;
+    /** Decoded text retained per part. Default 1 MiB. */
+    maxTextPartBytes?: number;
+    /** Per file timeout in milliseconds. Default 30000. */
+    timeoutMs?: number;
+    /** Rows buffered before a database write. Default 500. */
+    batchSize?: number;
+    /** Filename filter. Default matches .eml case insensitively. */
+    pattern?: RegExp;
+    /** Called with progress updates. */
+    onProgress?: (progress: ScanProgress) => void;
+    /** Progress callback interval in milliseconds. Default 1000. */
+    progressIntervalMs?: number;
+}
+export interface ScanProgress {
+    seen: number;
+    total: number;
+    processed: number;
+    failed: number;
+    skipped: number;
+    elapsedMs: number;
+    filesPerSecond: number;
+    etaMs: number | null;
+    rssBytes: number;
+}
+export interface ScanSummary extends ScanProgress {
+    runId: number;
+    status: 'completed' | 'interrupted';
+}
+interface Candidate {
+    relPath: string;
+    size: number;
+    mtime: number;
+}
+/**
+ * Walk a directory tree and yield candidate files.
+ *
+ * Uses opendir so that a directory with 320k entries is streamed rather than materialised as one
+ * large array by readdir. Unreadable entries are skipped silently: a permission error on one file
+ * is not a reason to abort a corpus walk.
+ */
+export declare function walkFiles(root: string, pattern: RegExp): AsyncGenerator<Candidate>;
+/**
+ * Select a deterministic sample.
+ *
+ * Each path is mapped to a uniform value by hashing it with the seed, and the lowest `count` values
+ * win. The result depends only on the path set and the seed, never on iteration order, so a sample
+ * is reproducible across machines and across partial runs.
+ */
+export declare function selectSample(candidates: readonly Candidate[], count: number, seed: string): Candidate[];
+export declare function scanCorpus(options: ScanOptions): Promise<ScanSummary>;
+export {};
+//# sourceMappingURL=scanner.d.ts.map

package/dist/corpus/scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.d.ts","sourceRoot":"","sources":["../../src/corpus/scanner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH,MAAM,WAAW,WAAW;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sFAAsF;IACtF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IAC9C,gEAAgE;IAChE,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAY,SAAQ,YAAY;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,WAAW,GAAG,aAAa,CAAC;CACvC;AAED,UAAU,SAAS;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;GAMG;AACH,wBAAuB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAgCzF;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CAOvG;AAMD,wBAAsB,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAqL3E"}

package/dist/corpus/scanner.js

@@ -0,0 +1,256 @@
+"use strict";
+/**
+ * Streaming corpus scanner.
+ *
+ * Requirements this design answers, in order of importance:
+ *   - one malformed message must never stop a run, so every failure becomes a row
+ *   - bounded memory, so files are streamed and results are flushed in batches
+ *   - resumable, so an interrupted run skips what it already recorded
+ *   - deterministic, so the same seed always produces the same sample
+ *   - no message content in any log line
+ *
+ * Backpressure is driven from here rather than inside the pool: the scanner keeps a bounded window
+ * of in flight tasks and awaits a completion before dispatching more. That keeps peak memory
+ * proportional to the window size rather than to the corpus size.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.walkFiles = walkFiles;
+exports.selectSample = selectSample;
+exports.scanCorpus = scanCorpus;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_os_1 = __importDefault(require("node:os"));
+const hash_1 = require("../util/hash");
+const worker_pool_1 = require("../util/worker-pool");
+const store_1 = require("./store");
+/**
+ * Walk a directory tree and yield candidate files.
+ *
+ * Uses opendir so that a directory with 320k entries is streamed rather than materialised as one
+ * large array by readdir. Unreadable entries are skipped silently: a permission error on one file
+ * is not a reason to abort a corpus walk.
+ */
+async function* walkFiles(root, pattern) {
+    const stack = [''];
+    while (stack.length) {
+        const relDir = stack.pop();
+        const absDir = node_path_1.default.join(root, relDir);
+        let dir;
+        try {
+            dir = node_fs_1.default.opendirSync(absDir);
+        }
+        catch {
+            continue;
+        }
+        try {
+            for await (const entry of dir) {
+                const relPath = relDir ? node_path_1.default.join(relDir, entry.name) : entry.name;
+                if (entry.isDirectory()) {
+                    stack.push(relPath);
+                    continue;
+                }
+                if (!entry.isFile() || !pattern.test(entry.name)) {
+                    continue;
+                }
+                try {
+                    const stat = node_fs_1.default.statSync(node_path_1.default.join(root, relPath));
+                    yield { relPath, size: stat.size, mtime: Math.floor(stat.mtimeMs) };
+                }
+                catch {
+                    // Vanished between readdir and stat. Nothing to record.
+                }
+            }
+        }
+        catch {
+            // Directory iteration failed midway. Keep whatever was yielded and move on.
+        }
+    }
+}
+/**
+ * Select a deterministic sample.
+ *
+ * Each path is mapped to a uniform value by hashing it with the seed, and the lowest `count` values
+ * win. The result depends only on the path set and the seed, never on iteration order, so a sample
+ * is reproducible across machines and across partial runs.
+ */
+function selectSample(candidates, count, seed) {
+    if (!count || count >= candidates.length) {
+        return [...candidates];
+    }
+    const scored = candidates.map(candidate => ({ candidate, score: (0, hash_1.deterministicUnit)(seed, candidate.relPath) }));
+    scored.sort((a, b) => a.score - b.score || (a.candidate.relPath < b.candidate.relPath ? -1 : 1));
+    return scored.slice(0, count).map(item => item.candidate);
+}
+function resolveWorkerPath() {
+    return node_path_1.default.join(__dirname, 'worker.js');
+}
+async function scanCorpus(options) {
+    const root = node_path_1.default.resolve(options.root);
+    const pattern = options.pattern ?? /\.eml$/i;
+    const concurrency = Math.max(1, options.concurrency ?? Math.max(1, node_os_1.default.availableParallelism() - 2));
+    const batchSize = options.batchSize ?? 500;
+    const timeoutMs = options.timeoutMs ?? 30000;
+    const maxFileBytes = options.maxFileBytes ?? 32 * 1024 * 1024;
+    const maxTextPartBytes = options.maxTextPartBytes ?? 1024 * 1024;
+    const progressIntervalMs = options.progressIntervalMs ?? 1000;
+    const resume = options.resume !== false;
+    const store = new store_1.CorpusStore(options.dbPath);
+    // The candidate list is collected first so that sampling is deterministic and progress can
+    // report a real total. Only path, size and mtime are held, roughly 100 bytes per file, which is
+    // about 32 MB for the full corpus. Message bodies are never held.
+    const candidates = [];
+    for await (const candidate of walkFiles(root, pattern)) {
+        candidates.push(candidate);
+    }
+    candidates.sort((a, b) => (a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0));
+    const selected = options.sample ? selectSample(candidates, options.sample, options.seed ?? 'default') : candidates;
+    const runId = store.startRun({
+        root,
+        concurrency,
+        sample: options.sample ?? null,
+        seed: options.seed ?? null,
+        resume,
+        maxFileBytes,
+        maxTextPartBytes,
+        timeoutMs,
+        total: selected.length
+    });
+    const pool = new worker_pool_1.WorkerPool({
+        workerPath: resolveWorkerPath(),
+        size: concurrency,
+        taskTimeoutMs: timeoutMs,
+        // Generous relative to the working set of one message, tight enough that the whole pool
+        // stays well under a gigabyte.
+        maxOldGenerationSizeMb: 192,
+        workerData: { root, salt: store.salt, maxFileBytes, maxTextPartBytes }
+    });
+    pool.on('worker-error', () => {
+        // Reported through the failed counter. Worker errors carry no message content, but there is
+        // nothing actionable to print per occurrence during a long run.
+    });
+    const started = Date.now();
+    let seen = 0;
+    let processed = 0;
+    let failed = 0;
+    let skipped = 0;
+    let interrupted = false;
+    let batch = [];
+    const flush = () => {
+        if (!batch.length) {
+            return;
+        }
+        const rows = batch;
+        batch = [];
+        store.writeBatch(rows, runId);
+    };
+    const progress = () => {
+        const elapsedMs = Date.now() - started;
+        const done = processed + failed;
+        const rate = elapsedMs > 0 ? (done / elapsedMs) * 1000 : 0;
+        const remaining = selected.length - seen;
+        return {
+            seen,
+            total: selected.length,
+            processed,
+            failed,
+            skipped,
+            elapsedMs,
+            filesPerSecond: rate,
+            etaMs: rate > 0 ? (remaining / rate) * 1000 : null,
+            rssBytes: process.memoryUsage.rss()
+        };
+    };
+    let lastProgress = 0;
+    const maybeReport = () => {
+        if (!options.onProgress) {
+            return;
+        }
+        const now = Date.now();
+        if (now - lastProgress >= progressIntervalMs) {
+            lastProgress = now;
+            options.onProgress(progress());
+        }
+    };
+    const onSignal = () => {
+        interrupted = true;
+    };
+    process.on('SIGINT', onSignal);
+    process.on('SIGTERM', onSignal);
+    const window = concurrency * 4;
+    const inFlight = new Set();
+    try {
+        for (const candidate of selected) {
+            if (interrupted) {
+                break;
+            }
+            seen++;
+            if (resume && store.isProcessed(candidate.relPath, candidate.size, candidate.mtime)) {
+                skipped++;
+                maybeReport();
+                continue;
+            }
+            const task = { relPath: candidate.relPath, size: candidate.size, mtime: candidate.mtime };
+            const promise = pool
+                .submit(task)
+                .then(result => {
+                batch.push({ file: result.file, message: result.message });
+                if (result.file.status === 'ok') {
+                    processed++;
+                }
+                else {
+                    failed++;
+                }
+            })
+                .catch((err) => {
+                // A rejected task means the worker died or timed out. Record it and continue.
+                failed++;
+                batch.push({
+                    file: {
+                        path: candidate.relPath,
+                        size: candidate.size,
+                        mtime: candidate.mtime,
+                        sha256: null,
+                        status: err.name === 'WorkerTimeoutError' ? 'timeout' : 'failed',
+                        errorCode: err.name === 'WorkerTimeoutError' ? 'timeout' : 'worker_error',
+                        errorDetail: err.message.slice(0, 300),
+                        durationMs: null
+                    },
+                    message: null
+                });
+            })
+                .then(() => {
+                if (batch.length >= batchSize) {
+                    flush();
+                }
+                maybeReport();
+            });
+            inFlight.add(promise);
+            void promise.finally(() => inFlight.delete(promise));
+            if (inFlight.size >= window) {
+                await Promise.race(inFlight);
+            }
+        }
+        await Promise.all(inFlight);
+        flush();
+    }
+    finally {
+        process.off('SIGINT', onSignal);
+        process.off('SIGTERM', onSignal);
+        await pool.close();
+        flush();
+        const final = progress();
+        store.finishRun(runId, interrupted ? 'interrupted' : 'completed', {
+            seen: final.seen,
+            processed: final.processed,
+            failed: final.failed,
+            skipped: final.skipped
+        });
+        store.close();
+    }
+    const final = progress();
+    return { ...final, runId, status: interrupted ? 'interrupted' : 'completed' };
+}
+//# sourceMappingURL=scanner.js.map

package/dist/corpus/scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.js","sourceRoot":"","sources":["../../src/corpus/scanner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;AAmEH,8BAgCC;AASD,oCAOC;AAMD,gCAqLC;AA5SD,sDAAyB;AACzB,0DAA6B;AAC7B,sDAAyB;AACzB,uCAAiD;AACjD,qDAAiD;AACjD,mCAAoD;AAqDpD;;;;;;GAMG;AACI,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,IAAY,EAAE,OAAe;IAC1D,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAY,CAAC;QACrC,MAAM,MAAM,GAAG,mBAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACD,GAAG,GAAG,iBAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACL,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,mBAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;gBACpE,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACpB,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/C,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC;oBACD,MAAM,IAAI,GAAG,iBAAE,CAAC,QAAQ,CAAC,mBAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;oBACnD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxE,CAAC;gBAAC,MAAM,CAAC;oBACL,wDAAwD;gBAC5D,CAAC;YACL,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,4EAA4E;QAChF,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,YAAY,CAAC,UAAgC,EAAE,KAAa,EAAE,IAAY;IACtF,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAA,wBAAiB,EAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/G,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,iBAAiB;IACtB,OAAO,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;AAC7C,CAAC;AAEM,KAAK,UAAU,UAAU,CAAC,OAAoB;IACjD,MAAM,IAAI,GAAG,mBAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC;IAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAE,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAC9D,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,IAAI,GAAG,IAAI,CAAC;IACjE,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC;IAExC,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9C,2FAA2F;IAC3F,gGAAgG;IAChG,kEAAkE;IAClE,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QACrD,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAExF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEnH,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC;QACzB,IAAI;QACJ,WAAW;QACX,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI;QAC1B,MAAM;QACN,YAAY;QACZ,gBAAgB;QAChB,SAAS;QACT,KAAK,EAAE,QAAQ,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,wBAAU,CAAuB;QAC9C,UAAU,EAAE,iBAAiB,EAAE;QAC/B,IAAI,EAAE,WAAW;QACjB,aAAa,EAAE,SAAS;QACxB,wFAAwF;QACxF,+BAA+B;QAC/B,sBAAsB,EAAE,GAAG;QAC3B,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,gBAAgB,EAAE;KACzE,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACzB,4FAA4F;QAC5F,gEAAgE;IACpE,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,KAAK,GAAc,EAAE,CAAC;IAE1B,MAAM,KAAK,GAAG,GAAG,EAAE;QACf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC;QACnB,KAAK,GAAG,EAAE,CAAC;QACX,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAiB,EAAE;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACvC,MAAM,IAAI,GAAG,SAAS,GAAG,MAAM,CAAC;QAChC,MAAM,IAAI,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC;QACzC,OAAO;YACH,IAAI;YACJ,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,SAAS;YACT,MAAM;YACN,OAAO;YACP,SAAS;YACT,cAAc,EAAE,IAAI;YACpB,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI;YAClD,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE;SACtC,CAAC;IACN,CAAC,CAAC;IAEF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,YAAY,IAAI,kBAAkB,EAAE,CAAC;YAC3C,YAAY,GAAG,GAAG,CAAC;YACnB,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAG,EAAE;QAClB,WAAW,GAAG,IAAI,CAAC;IACvB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhC,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE1C,IAAI,CAAC;QACD,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM;YACV,CAAC;YACD,IAAI,EAAE,CAAC;YAEP,IAAI,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClF,OAAO,EAAE,CAAC;gBACV,WAAW,EAAE,CAAC;gBACd,SAAS;YACb,CAAC;YAED,MAAM,IAAI,GAAa,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;YACpG,MAAM,OAAO,GAAG,IAAI;iBACf,MAAM,CAAC,IAAI,CAAC;iBACZ,IAAI,CAAC,MAAM,CAAC,EAAE;gBACX,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;oBAC9B,SAAS,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,MAAM,EAAE,CAAC;gBACb,CAAC;YACL,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;gBAClB,8EAA8E;gBAC9E,MAAM,EAAE,CAAC;gBACT,KAAK,CAAC,IAAI,CAAC;oBACP,IAAI,EAAE;wBACF,IAAI,EAAE,SAAS,CAAC,OAAO;wBACvB,IAAI,EAAE,SAAS,CAAC,IAAI;wBACpB,KAAK,EAAE,SAAS,CAAC,KAAK;wBACtB,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,GAAG,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;wBAChE,SAAS,EAAE,GAAG,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc;wBACzE,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBACtC,UAAU,EAAE,IAAI;qBACnB;oBACD,OAAO,EAAE,IAAI;iBAChB,CAAC,CAAC;YACP,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC5B,KAAK,EAAE,CAAC;gBACZ,CAAC;gBACD,WAAW,EAAE,CAAC;YAClB,CAAC,CAAC,CAAC;YAEP,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAErD,IAAI,QAAQ,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5B,KAAK,EAAE,CAAC;IACZ,CAAC;YAAS,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,KAAK,EAAE,CAAC;QACR,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;QACzB,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAE;YAC9D,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;IACzB,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;AAClF,CAAC"}

package/dist/corpus/stats.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Aggregate corpus statistics.
+ *
+ * Everything here is a count, a ratio or a percentile. No subject, address or body text is read,
+ * and the only free text that reaches the report is a MIME structure signature and detector ids,
+ * both of which are drawn from a closed vocabulary. That is what makes the report safe to paste
+ * into a document or an issue.
+ */
+import { type CorpusStore } from './store';
+export interface Distribution {
+    label: string;
+    count: number;
+    share: number;
+}
+export interface CorpusStats {
+    generatedAt: string;
+    files: {
+        total: number;
+        ok: number;
+        failed: number;
+        oversize: number;
+        timeout: number;
+        skipped: number;
+    };
+    errors: Distribution[];
+    sizeBytes: {
+        p10: number;
+        p50: number;
+        p90: number;
+        p99: number;
+        max: number;
+        total: number;
+    };
+    parseWarnings: Distribution[];
+    bodyKind: Distribution[];
+    structures: Distribution[];
+    charsets: Distribution[];
+    encodings: Distribution[];
+    traffic: Distribution[];
+    clients: Distribution[];
+    clientSources: Distribution[];
+    subjectPrefix: Distribution[];
+    replyStyle: Distribution[];
+    quoteDepth: Distribution[];
+    markers: Distribution[];
+    htmlHints: Distribution[];
+    languages: Distribution[];
+    duplicates: {
+        distinctSha256: number;
+        filesInDuplicateGroups: number;
+        distinctSimhash: number;
+        filesInNearDuplicateGroups: number;
+    };
+    threading: {
+        withMessageId: number;
+        withInReplyTo: number;
+        parentResolvedInCorpus: number;
+        withReferences: number;
+        ancestorResolvedInCorpus: number;
+    };
+    humanSlice: {
+        total: number;
+        replies: number;
+        repliesWithParentInCorpus: number;
+        bothRepresentations: number;
+        quoteMarkingDisagrees: number;
+    };
+    contentAgreement: {
+        messagesWithBoth: number;
+        quoteMarkingDisagrees: number;
+    };
+}
+export declare function computeStats(store: CorpusStore): CorpusStats;
+/** Render the statistics as a Markdown report. Safe to share: contains no message content. */
+export declare function renderReport(stats: CorpusStats, root: string): string;
+//# sourceMappingURL=stats.d.ts.map

package/dist/corpus/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/corpus/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,WAAW,YAAY;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE;QACH,KAAK,EAAE,MAAM,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9F,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,UAAU,EAAE;QACR,cAAc,EAAE,MAAM,CAAC;QACvB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,eAAe,EAAE,MAAM,CAAC;QACxB,0BAA0B,EAAE,MAAM,CAAC;KACtC,CAAC;IACF,SAAS,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,cAAc,EAAE,MAAM,CAAC;QACvB,wBAAwB,EAAE,MAAM,CAAC;KACpC,CAAC;IACF,UAAU,EAAE;QACR,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,yBAAyB,EAAE,MAAM,CAAC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,qBAAqB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,gBAAgB,EAAE;QACd,gBAAgB,EAAE,MAAM,CAAC;QACzB,qBAAqB,EAAE,MAAM,CAAC;KACjC,CAAC;CACL;AA8DD,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CA8G5D;AAkCD,8FAA8F;AAC9F,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CA+GrE"}