@postalsys/email-content 0.0.1

package/dist/cli/benchmark.js

@@ -0,0 +1,172 @@
+"use strict";
+/**
+ * `email-content benchmark` subcommand.
+ *
+ * Two jobs in one pass over a sample of the corpus: measure throughput and memory, and count the
+ * catastrophic outcomes that matter more than any average. A benchmark that only reported
+ * milliseconds per message would miss the failure this project exists to prevent.
+ *
+ * The catastrophic checks here are structural and need no labels, so they can run over hundreds of
+ * thousands of messages. They are a floor, not a substitute for the labelled evaluation in Phase 4.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.benchmarkCommand = benchmarkCommand;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const index_1 = require("../index");
+const store_1 = require("../corpus/store");
+const args_1 = require("./args");
+const ANCHOR_REASONS = new Set([
+    'preceded_by_reply_header',
+    'preceded_by_forward_header',
+    'inside_quote_container',
+    'outlook_reply_separator',
+    'matched_parent_message',
+    'matched_thread_ancestor'
+]);
+function percentile(values, fraction) {
+    if (!values.length) {
+        return 0;
+    }
+    const index = Math.min(values.length - 1, Math.max(0, Math.floor(values.length * fraction)));
+    return values[index] ?? 0;
+}
+async function benchmarkCommand(args) {
+    const dbPath = (0, args_1.flagString)(args, 'db', 'data/corpus.sqlite');
+    if (!node_fs_1.default.existsSync(dbPath)) {
+        process.stderr.write(`no corpus database at ${dbPath}. Run "corpus scan" first.\n`);
+        return 2;
+    }
+    const limit = (0, args_1.flagNumber)(args, 'sample', 2000);
+    const traffic = (0, args_1.flagString)(args, 'traffic', 'human');
+    const onlyReplies = (0, args_1.flagString)(args, 'replies', 'true') !== 'false';
+    const store = new store_1.CorpusStore(dbPath);
+    const root = store.queryOne('SELECT root FROM runs ORDER BY id DESC LIMIT 1')?.root;
+    if (!root) {
+        process.stderr.write('no completed scan run found\n');
+        store.close();
+        return 2;
+    }
+    // Ordering by path keeps the selection deterministic for a given database and filter.
+    const where = ["f.status = 'ok'"];
+    if (traffic !== 'all') {
+        where.push(`m.traffic = '${traffic.replace(/'/g, '')}'`);
+    }
+    if (onlyReplies) {
+        where.push('m.is_reply = 1');
+    }
+    const rows = store.query(`SELECT m.path AS path FROM messages m JOIN files f ON f.path = m.path WHERE ${where.join(' AND ')} ORDER BY m.path LIMIT ?`, limit);
+    store.close();
+    if (!rows.length) {
+        process.stderr.write('no matching messages\n');
+        return 1;
+    }
+    const counters = {
+        messages: 0,
+        errors: 0,
+        emptyFromNonEmpty: 0,
+        everythingHidden: 0,
+        onlySignatureVisible: 0,
+        hiddenWithoutAnchor: 0,
+        forwardedHidden: 0,
+        hiddenChars: 0,
+        visibleChars: 0,
+        labels: new Map(),
+        durations: []
+    };
+    const rssStart = process.memoryUsage.rss();
+    let rssPeak = rssStart;
+    const started = Date.now();
+    for (const row of rows) {
+        let raw;
+        try {
+            raw = node_fs_1.default.readFileSync(node_path_1.default.join(root, row.path));
+        }
+        catch {
+            counters.errors++;
+            continue;
+        }
+        const t0 = process.hrtime.bigint();
+        let result;
+        try {
+            result = await (0, index_1.extract)(raw);
+        }
+        catch {
+            counters.errors++;
+            continue;
+        }
+        counters.durations.push(Number(process.hrtime.bigint() - t0) / 1e6);
+        counters.messages++;
+        const visible = result.segments;
+        const hidden = result.hiddenSegments;
+        const visibleChars = visible.reduce((sum, segment) => sum + segment.text.length, 0);
+        const hiddenChars = hidden.reduce((sum, segment) => sum + segment.text.length, 0);
+        counters.visibleChars += visibleChars;
+        counters.hiddenChars += hiddenChars;
+        for (const segment of [...visible, ...hidden]) {
+            counters.labels.set(segment.type, (counters.labels.get(segment.type) ?? 0) + 1);
+        }
+        const hadContent = visibleChars + hiddenChars > 0;
+        if (hadContent && visibleChars === 0) {
+            counters.everythingHidden++;
+        }
+        if (hadContent && !visible.length) {
+            counters.emptyFromNonEmpty++;
+        }
+        if (visible.length > 0 && visible.every(segment => segment.type === 'SIGNATURE' || segment.type === 'DISCLAIMER')) {
+            counters.onlySignatureVisible++;
+        }
+        for (const segment of hidden) {
+            if (!segment.reasons.some(reason => ANCHOR_REASONS.has(reason))) {
+                counters.hiddenWithoutAnchor++;
+            }
+            if (segment.type === 'FORWARDED_CONTENT') {
+                counters.forwardedHidden++;
+            }
+        }
+        if (counters.messages % 250 === 0) {
+            rssPeak = Math.max(rssPeak, process.memoryUsage.rss());
+        }
+    }
+    rssPeak = Math.max(rssPeak, process.memoryUsage.rss());
+    const elapsed = Date.now() - started;
+    counters.durations.sort((a, b) => a - b);
+    const out = [];
+    const pct = (value) => `${((100 * value) / Math.max(1, counters.messages)).toFixed(2)}%`;
+    out.push(`sample: ${counters.messages} messages (traffic=${traffic}, replies-only=${onlyReplies})`);
+    out.push(`errors: ${counters.errors}`);
+    out.push('');
+    out.push('THROUGHPUT');
+    out.push(`  total            ${(elapsed / 1000).toFixed(1)}s`);
+    out.push(`  rate             ${((counters.messages / elapsed) * 1000).toFixed(0)} messages/s (single threaded)`);
+    out.push(`  per message p50  ${percentile(counters.durations, 0.5).toFixed(2)} ms`);
+    out.push(`  per message p90  ${percentile(counters.durations, 0.9).toFixed(2)} ms`);
+    out.push(`  per message p99  ${percentile(counters.durations, 0.99).toFixed(2)} ms`);
+    out.push(`  per message max  ${percentile(counters.durations, 1).toFixed(2)} ms`);
+    out.push(`  rss delta        ${((rssPeak - rssStart) / (1024 * 1024)).toFixed(0)} MB`);
+    out.push('');
+    out.push('CONTENT');
+    const totalChars = counters.visibleChars + counters.hiddenChars;
+    out.push(`  visible characters ${counters.visibleChars.toLocaleString('en-US')} (${((100 * counters.visibleChars) / Math.max(1, totalChars)).toFixed(1)}%)`);
+    out.push(`  hidden characters  ${counters.hiddenChars.toLocaleString('en-US')} (${((100 * counters.hiddenChars) / Math.max(1, totalChars)).toFixed(1)}%)`);
+    out.push('');
+    out.push('SEGMENT LABELS');
+    for (const [label, count] of [...counters.labels].sort((a, b) => b[1] - a[1])) {
+        out.push(`  ${label.padEnd(18)} ${String(count).padStart(7)}`);
+    }
+    out.push('');
+    out.push('CATASTROPHIC CHECKS (structural, no labels required)');
+    out.push(`  empty extraction from non empty message  ${counters.emptyFromNonEmpty} (${pct(counters.emptyFromNonEmpty)})`);
+    out.push(`  all content hidden                       ${counters.everythingHidden} (${pct(counters.everythingHidden)})`);
+    out.push(`  only a footer left visible               ${counters.onlySignatureVisible} (${pct(counters.onlySignatureVisible)})`);
+    out.push(`  hidden without a structural anchor       ${counters.hiddenWithoutAnchor} (${pct(counters.hiddenWithoutAnchor)})`);
+    out.push(`  forwarded content hidden                 ${counters.forwardedHidden} (${pct(counters.forwardedHidden)})`);
+    process.stdout.write(`${out.join('\n')}\n`);
+    // A non zero exit makes this usable as a regression gate.
+    const failed = counters.emptyFromNonEmpty + counters.everythingHidden + counters.hiddenWithoutAnchor + counters.forwardedHidden;
+    return failed === 0 ? 0 : 1;
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/cli/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/cli/benchmark.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;AAwCH,4CA0JC;AAhMD,sDAAyB;AACzB,0DAA6B;AAC7B,oCAAmC;AACnC,2CAA8C;AAE9C,iCAAiE;AAgBjE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC3B,0BAA0B;IAC1B,4BAA4B;IAC5B,wBAAwB;IACxB,yBAAyB;IACzB,wBAAwB;IACxB,yBAAyB;CAC5B,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,MAAyB,EAAE,QAAgB;IAC3D,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7F,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAEM,KAAK,UAAU,gBAAgB,CAAC,IAAgB;IACnD,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC5D,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,8BAA8B,CAAC,CAAC;QACpF,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,KAAK,OAAO,CAAC;IAEpE,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAmB,gDAAgD,CAAC,EAAE,IAAI,CAAC;IACtG,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACtD,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACb,CAAC;IAED,sFAAsF;IACtF,MAAM,KAAK,GAAa,CAAC,iBAAiB,CAAC,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,WAAW,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CACpB,+EAA+E,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,0BAA0B,EAC5H,KAAK,CACR,CAAC;IACF,KAAK,CAAC,KAAK,EAAE,CAAC;IAEd,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,QAAQ,GAAa;QACvB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,CAAC;QACT,iBAAiB,EAAE,CAAC;QACpB,gBAAgB,EAAE,CAAC;QACnB,oBAAoB,EAAE,CAAC;QACvB,mBAAmB,EAAE,CAAC;QACtB,eAAe,EAAE,CAAC;QAClB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,IAAI,GAAG,EAAE;QACjB,SAAS,EAAE,EAAE;KAChB,CAAC;IAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;IAC3C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACD,MAAM,GAAG,MAAM,IAAA,eAAO,EAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QACD,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;QACpE,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAEpB,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;QACrC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClF,QAAQ,CAAC,YAAY,IAAI,YAAY,CAAC;QACtC,QAAQ,CAAC,WAAW,IAAI,WAAW,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;YAC5C,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,UAAU,GAAG,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC;QAClD,IAAI,UAAU,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAChC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;YAChH,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QACpC,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAC9D,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACnC,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;gBACvC,QAAQ,CAAC,eAAe,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3D,CAAC;IACL,CAAC;IAED,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACrC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjG,GAAG,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,QAAQ,sBAAsB,OAAO,kBAAkB,WAAW,GAAG,CAAC,CAAC;IACpG,GAAG,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACvB,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/D,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC;IACjH,GAAG,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACpF,GAAG,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACpF,GAAG,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrF,GAAG,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAClF,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACvF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpB,MAAM,UAAU,GAAG,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,WAAW,CAAC;IAChE,GAAG,CAAC,IAAI,CACJ,wBAAwB,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACrJ,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3J,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,8CAA8C,QAAQ,CAAC,iBAAiB,KAAK,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;IAC1H,GAAG,CAAC,IAAI,CAAC,8CAA8C,QAAQ,CAAC,gBAAgB,KAAK,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACxH,GAAG,CAAC,IAAI,CAAC,8CAA8C,QAAQ,CAAC,oBAAoB,KAAK,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAChI,GAAG,CAAC,IAAI,CAAC,8CAA8C,QAAQ,CAAC,mBAAmB,KAAK,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC9H,GAAG,CAAC,IAAI,CAAC,8CAA8C,QAAQ,CAAC,eAAe,KAAK,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAEtH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE5C,0DAA0D;IAC1D,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,GAAG,QAAQ,CAAC,gBAAgB,GAAG,QAAQ,CAAC,mBAAmB,GAAG,QAAQ,CAAC,eAAe,CAAC;IAChI,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC"}

package/dist/cli/corpus.d.ts

@@ -0,0 +1,9 @@
+/**
+ * `email-content corpus ...` subcommands.
+ *
+ * Progress goes to stderr so that stdout stays clean for piping. No line printed here can contain
+ * message content: only counts, rates and file paths relative to the corpus root.
+ */
+import { type ParsedArgs } from './args';
+export declare function corpusCommand(sub: string | undefined, args: ParsedArgs): Promise<number>;
+//# sourceMappingURL=corpus.d.ts.map

package/dist/cli/corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../src/cli/corpus.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,OAAO,EAAwD,KAAK,UAAU,EAAE,MAAM,QAAQ,CAAC;AAoI/F,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAY9F"}

package/dist/cli/corpus.js

@@ -0,0 +1,152 @@
+"use strict";
+/**
+ * `email-content corpus ...` subcommands.
+ *
+ * Progress goes to stderr so that stdout stays clean for piping. No line printed here can contain
+ * message content: only counts, rates and file paths relative to the corpus root.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.corpusCommand = corpusCommand;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const scanner_1 = require("../corpus/scanner");
+const stats_1 = require("../corpus/stats");
+const store_1 = require("../corpus/store");
+const args_1 = require("./args");
+const DEFAULT_DB = 'data/corpus.sqlite';
+function formatDuration(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    if (minutes < 60) {
+        return `${minutes}m ${seconds % 60}s`;
+    }
+    return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
+}
+function renderProgress(progress) {
+    const pct = progress.total ? ((100 * progress.seen) / progress.total).toFixed(1) : '0.0';
+    const eta = progress.etaMs === null ? 'unknown' : formatDuration(progress.etaMs);
+    const rss = (progress.rssBytes / (1024 * 1024)).toFixed(0);
+    return (`  ${progress.seen.toLocaleString('en-US')}/${progress.total.toLocaleString('en-US')} (${pct}%)  ` +
+        `ok=${progress.processed.toLocaleString('en-US')} failed=${progress.failed} skipped=${progress.skipped}  ` +
+        `${progress.filesPerSecond.toFixed(0)}/s  eta ${eta}  rss ${rss}MB`);
+}
+async function scan(args) {
+    const root = args.positional[0];
+    if (!root) {
+        process.stderr.write('usage: email-content corpus scan <folder> [--db path] [--sample N] [--seed S] [--concurrency N] [--no-resume]\n');
+        return 2;
+    }
+    if (!node_fs_1.default.existsSync(root)) {
+        process.stderr.write(`corpus root not found: ${root}\n`);
+        return 2;
+    }
+    const dbPath = (0, args_1.flagString)(args, 'db', DEFAULT_DB);
+    const isTty = process.stderr.isTTY;
+    let lastLength = 0;
+    process.stderr.write(`Scanning ${node_path_1.default.resolve(root)}\n`);
+    process.stderr.write(`Database ${node_path_1.default.resolve(dbPath)}\n`);
+    const summary = await (0, scanner_1.scanCorpus)({
+        root,
+        dbPath,
+        sample: (0, args_1.flagNumber)(args, 'sample', 0),
+        seed: (0, args_1.flagString)(args, 'seed', 'default'),
+        concurrency: (0, args_1.flagNumber)(args, 'concurrency', 0) || undefined,
+        resume: (0, args_1.flagBool)(args, 'resume', true),
+        maxFileBytes: (0, args_1.flagNumber)(args, 'max-file-mb', 32) * 1024 * 1024,
+        timeoutMs: (0, args_1.flagNumber)(args, 'timeout', 30000),
+        batchSize: (0, args_1.flagNumber)(args, 'batch', 500),
+        onProgress: progress => {
+            const line = renderProgress(progress);
+            if (isTty) {
+                process.stderr.write(`\r${line.padEnd(lastLength)}`);
+                lastLength = line.length;
+            }
+            else {
+                process.stderr.write(`${line}\n`);
+            }
+        }
+    });
+    if (isTty) {
+        process.stderr.write('\n');
+    }
+    process.stderr.write(`${summary.status}: ${summary.processed.toLocaleString('en-US')} parsed, ${summary.failed} failed, ${summary.skipped} skipped in ${formatDuration(summary.elapsedMs)} ` +
+        `(${summary.filesPerSecond.toFixed(0)} files/s)\n`);
+    return summary.status === 'completed' ? 0 : 1;
+}
+function stats(args) {
+    const dbPath = (0, args_1.flagString)(args, 'db', DEFAULT_DB);
+    if (!node_fs_1.default.existsSync(dbPath)) {
+        process.stderr.write(`no corpus database at ${dbPath}. Run "corpus scan" first.\n`);
+        return 2;
+    }
+    const store = new store_1.CorpusStore(dbPath);
+    try {
+        const computed = (0, stats_1.computeStats)(store);
+        if ((0, args_1.flagBool)(args, 'json', false)) {
+            process.stdout.write(`${JSON.stringify(computed, null, 2)}\n`);
+            return 0;
+        }
+        const root = store.queryOne('SELECT root FROM runs ORDER BY id DESC LIMIT 1')?.root ?? 'unknown';
+        const report = (0, stats_1.renderReport)(computed, root);
+        const out = (0, args_1.flagOptionalString)(args, 'out');
+        if (out) {
+            node_fs_1.default.mkdirSync(node_path_1.default.dirname(node_path_1.default.resolve(out)), { recursive: true });
+            node_fs_1.default.writeFileSync(out, report);
+            process.stderr.write(`wrote ${out}\n`);
+        }
+        else {
+            process.stdout.write(report);
+        }
+        return 0;
+    }
+    finally {
+        store.close();
+    }
+}
+function threads(args) {
+    const dbPath = (0, args_1.flagString)(args, 'db', DEFAULT_DB);
+    if (!node_fs_1.default.existsSync(dbPath)) {
+        process.stderr.write(`no corpus database at ${dbPath}. Run "corpus scan" first.\n`);
+        return 2;
+    }
+    const store = new store_1.CorpusStore(dbPath);
+    try {
+        const rows = store.query(`
+            SELECT CASE
+                WHEN m.in_reply_to IS NULL THEN 'no-in-reply-to'
+                WHEN EXISTS (SELECT 1 FROM messages p WHERE p.message_id = m.in_reply_to) THEN 'parent-in-corpus'
+                ELSE 'parent-missing'
+            END AS label, COUNT(*) AS count
+            FROM messages m GROUP BY label
+        `);
+        const total = rows.reduce((sum, row) => sum + row.count, 0);
+        for (const row of rows.sort((a, b) => b.count - a.count)) {
+            const share = total ? ((100 * row.count) / total).toFixed(1) : '0.0';
+            process.stdout.write(`${row.label.padEnd(20)} ${String(row.count).padStart(8)}  ${share.padStart(5)}%\n`);
+        }
+        return 0;
+    }
+    finally {
+        store.close();
+    }
+}
+async function corpusCommand(sub, args) {
+    switch (sub) {
+        case 'scan':
+            return scan(args);
+        case 'stats':
+            return stats(args);
+        case 'threads':
+            return threads(args);
+        default:
+            process.stderr.write('usage: email-content corpus <scan|stats|threads> [options]\n');
+            return 2;
+    }
+}
+//# sourceMappingURL=corpus.js.map

package/dist/cli/corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../../src/cli/corpus.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;AA2IH,sCAYC;AArJD,sDAAyB;AACzB,0DAA6B;AAC7B,+CAAkE;AAClE,2CAA6D;AAC7D,2CAA8C;AAC9C,iCAA+F;AAE/F,MAAM,UAAU,GAAG,oBAAoB,CAAC;AAExC,SAAS,cAAc,CAAC,EAAU;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACtC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,GAAG,OAAO,GAAG,CAAC;IACzB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACzC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,GAAG,OAAO,KAAK,OAAO,GAAG,EAAE,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,OAAO,GAAG,EAAE,GAAG,CAAC;AAC3D,CAAC;AAED,SAAS,cAAc,CAAC,QAAsB;IAC1C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACzF,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACjF,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3D,OAAO,CACH,KAAK,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,GAAG,MAAM;QAClG,MAAM,QAAQ,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,WAAW,QAAQ,CAAC,MAAM,YAAY,QAAQ,CAAC,OAAO,IAAI;QAC1G,GAAG,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,GAAG,IAAI,CACtE,CAAC;AACN,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAgB;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iHAAiH,CAAC,CAAC;QACxI,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,IAAI,CAAC,CAAC;QACzD,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACnC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,mBAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAU,EAAC;QAC7B,IAAI;QACJ,MAAM;QACN,MAAM,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrC,IAAI,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC;QACzC,WAAW,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC,IAAI,SAAS;QAC5D,MAAM,EAAE,IAAA,eAAQ,EAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC;QACtC,YAAY,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,aAAa,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/D,SAAS,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC7C,SAAS,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC;QACzC,UAAU,EAAE,QAAQ,CAAC,EAAE;YACnB,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,KAAK,EAAE,CAAC;gBACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBACrD,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAChB,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,YAAY,OAAO,CAAC,MAAM,YAAY,OAAO,CAAC,OAAO,eAAe,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;QACnK,IAAI,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CACzD,CAAC;IACF,OAAO,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,KAAK,CAAC,IAAgB;IAC3B,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAClD,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,8BAA8B,CAAC,CAAC;QACpF,OAAO,CAAC,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,IAAA,oBAAY,EAAC,KAAK,CAAC,CAAC;QACrC,IAAI,IAAA,eAAQ,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/D,OAAO,CAAC,CAAC;QACb,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAmB,gDAAgD,CAAC,EAAE,IAAI,IAAI,SAAS,CAAC;QACnH,MAAM,MAAM,GAAG,IAAA,oBAAY,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAA,yBAAkB,EAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5C,IAAI,GAAG,EAAE,CAAC;YACN,iBAAE,CAAC,SAAS,CAAC,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,iBAAE,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,CAAC,CAAC;IACb,CAAC;YAAS,CAAC;QACP,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,IAAgB;IAC7B,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAClD,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,8BAA8B,CAAC,CAAC;QACpF,OAAO,CAAC,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAmC;;;;;;;SAO1D,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACrE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9G,CAAC;QACD,OAAO,CAAC,CAAC;IACb,CAAC;YAAS,CAAC;QACP,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,aAAa,CAAC,GAAuB,EAAE,IAAgB;IACzE,QAAQ,GAAG,EAAE,CAAC;QACV,KAAK,MAAM;YACP,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,KAAK,OAAO;YACR,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,SAAS;YACV,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;QACzB;YACI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YACrF,OAAO,CAAC,CAAC;IACjB,CAAC;AACL,CAAC"}

package/dist/cli/disagreements.d.ts

@@ -0,0 +1,14 @@
+/**
+ * `email-content disagreements` subcommand.
+ *
+ * The point of annotating with a model is not the agreement percentage, it is this list. Blocks
+ * where the deterministic baseline and the annotator disagree are where a human's attention is
+ * worth spending, and they sort naturally by cost: a block the baseline hid but the annotator calls
+ * authored is a candidate content loss bug, and everything else is a candidate tidiness bug.
+ *
+ * This command prints message content by design, because a reviewer has to read the block to judge
+ * it. It is the only command that does so besides `extract`.
+ */
+import { type ParsedArgs } from './args';
+export declare function disagreementsCommand(args: ParsedArgs): Promise<number>;
+//# sourceMappingURL=disagreements.d.ts.map

package/dist/cli/disagreements.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"disagreements.d.ts","sourceRoot":"","sources":["../../src/cli/disagreements.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,OAAO,EAA0B,KAAK,UAAU,EAAE,MAAM,QAAQ,CAAC;AA2CjE,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CA0F5E"}

package/dist/cli/disagreements.js

@@ -0,0 +1,133 @@
+"use strict";
+/**
+ * `email-content disagreements` subcommand.
+ *
+ * The point of annotating with a model is not the agreement percentage, it is this list. Blocks
+ * where the deterministic baseline and the annotator disagree are where a human's attention is
+ * worth spending, and they sort naturally by cost: a block the baseline hid but the annotator calls
+ * authored is a candidate content loss bug, and everything else is a candidate tidiness bug.
+ *
+ * This command prints message content by design, because a reviewer has to read the block to judge
+ * it. It is the only command that does so besides `extract`.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.disagreementsCommand = disagreementsCommand;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const rules_1 = require("../classify/rules");
+const store_1 = require("../corpus/store");
+const parse_1 = require("../mime/parse");
+const html_blocks_1 = require("../segment/html-blocks");
+const text_blocks_1 = require("../segment/text-blocks");
+const args_1 = require("./args");
+const DIRECTIONS = {
+    expensive: "baseline_label = 'REPLY_HISTORY' AND label IN ('NEW_CONTENT', 'INLINE_QUOTE')",
+    cheap: "label = 'REPLY_HISTORY' AND baseline_label IN ('NEW_CONTENT', 'INLINE_QUOTE')",
+    footers: "baseline_label = 'NEW_CONTENT' AND label IN ('SIGNATURE', 'DISCLAIMER')",
+    all: 'agrees = 0'
+};
+/** Rebuild the blocks of a message so the stored offsets can be resolved back to text. */
+async function blocksFor(root, relPath) {
+    const message = await (0, parse_1.parseMessage)(node_fs_1.default.readFileSync(node_path_1.default.join(root, relPath)));
+    let best = null;
+    for (const part of message.parts) {
+        if (!part.decoded || part.disposition === 'attachment') {
+            continue;
+        }
+        if (part.contentType === 'text/html') {
+            if (!best || best.representation === 'text' || part.decoded.text.length > best.text.length) {
+                best = { part: part.partId, text: part.decoded.text, representation: 'html' };
+            }
+        }
+        else if (part.contentType === 'text/plain' && (!best || (best.representation === 'text' && part.decoded.text.length > best.text.length))) {
+            best = { part: part.partId, text: part.decoded.text, representation: 'text' };
+        }
+    }
+    if (!best) {
+        return [];
+    }
+    return best.representation === 'html' ? (0, html_blocks_1.segmentHtmlBlocks)(best.text, best.part).blocks : (0, text_blocks_1.segmentTextBlocks)(best.text, best.part).blocks;
+}
+async function disagreementsCommand(args) {
+    const dbPath = (0, args_1.flagString)(args, 'db', 'data/corpus.sqlite');
+    if (!node_fs_1.default.existsSync(dbPath)) {
+        process.stderr.write(`no corpus database at ${dbPath}\n`);
+        return 2;
+    }
+    const direction = (0, args_1.flagString)(args, 'direction', 'expensive');
+    const clause = DIRECTIONS[direction];
+    if (!clause) {
+        process.stderr.write(`unknown direction: ${direction}. Use one of ${Object.keys(DIRECTIONS).join(', ')}\n`);
+        return 2;
+    }
+    const limit = (0, args_1.flagNumber)(args, 'limit', 20);
+    const context = (0, args_1.flagNumber)(args, 'context', 2);
+    const store = new store_1.CorpusStore(dbPath);
+    const root = store.queryOne('SELECT root FROM runs ORDER BY id DESC LIMIT 1')?.root;
+    if (!root) {
+        process.stderr.write('no completed scan run found\n');
+        store.close();
+        return 2;
+    }
+    const runFilter = (0, args_1.flagNumber)(args, 'run', 0);
+    const rows = store.query(`SELECT path, block_index, representation, label, confidence, baseline_label, baseline_conf, baseline_reasons, char_count
+         FROM annotations
+         WHERE ${clause} ${runFilter ? 'AND run_id = ' + runFilter : ''}
+         ORDER BY char_count DESC
+         LIMIT ?`, limit);
+    if (!rows.length) {
+        process.stdout.write(`no ${direction} disagreements recorded\n`);
+        store.close();
+        return 0;
+    }
+    const out = [];
+    out.push(`${rows.length} ${direction} disagreement(s), largest first`);
+    if (direction === 'expensive') {
+        out.push('The baseline treated these as reply history. The annotator thought the sender wrote them.');
+        out.push('Each one is a candidate content loss bug and should be judged by a human.');
+    }
+    out.push('');
+    const cache = new Map();
+    for (const row of rows) {
+        let blocks = cache.get(row.path);
+        if (!blocks) {
+            try {
+                blocks = await blocksFor(root, row.path);
+            }
+            catch {
+                blocks = [];
+            }
+            cache.set(row.path, blocks);
+        }
+        const block = blocks.find(candidate => candidate.index === row.block_index);
+        const decisions = blocks.length ? (0, rules_1.classifyBlocks)(blocks) : [];
+        out.push(`${row.path}  block #${row.block_index}  (${row.representation})`);
+        out.push(`  baseline: ${row.baseline_label} ${row.baseline_conf?.toFixed(2) ?? '?'}   annotator: ${row.label} ${row.confidence.toFixed(2)}`);
+        out.push(`  reasons:  ${row.baseline_reasons || 'none'}`);
+        if (block) {
+            out.push(`  depth:    quote=${block.features.quoteDepth} blockquote=${block.features.blockquoteDepth} kind=${block.kind}`);
+        }
+        // Neighbouring blocks are what make a decision judgeable: the same text means different
+        // things above and below an attribution line.
+        const from = Math.max(0, row.block_index - context);
+        const to = Math.min(blocks.length - 1, row.block_index + context);
+        for (let i = from; i <= to; i++) {
+            const neighbour = blocks[i];
+            if (!neighbour || neighbour.features.isAllBlank) {
+                continue;
+            }
+            const decision = decisions.find(candidate => candidate.block.index === i);
+            const marker = i === row.block_index ? '>>' : '  ';
+            const text = neighbour.content.replace(/\s+/g, ' ').slice(0, 110);
+            out.push(`  ${marker} #${String(i).padStart(2)} d${neighbour.features.quoteDepth} ${(decision?.label ?? '?').padEnd(17)} ${text}`);
+        }
+        out.push('');
+    }
+    process.stdout.write(`${out.join('\n')}\n`);
+    store.close();
+    return 0;
+}
+//# sourceMappingURL=disagreements.js.map

package/dist/cli/disagreements.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"disagreements.js","sourceRoot":"","sources":["../../src/cli/disagreements.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;AAqDH,oDA0FC;AA7ID,sDAAyB;AACzB,0DAA6B;AAC7B,6CAAmD;AACnD,2CAA8C;AAC9C,yCAA6C;AAC7C,wDAA2D;AAC3D,wDAA2D;AAE3D,iCAAiE;AAcjE,MAAM,UAAU,GAA2B;IACvC,SAAS,EAAE,+EAA+E;IAC1F,KAAK,EAAE,+EAA+E;IACtF,OAAO,EAAE,yEAAyE;IAClF,GAAG,EAAE,YAAY;CACpB,CAAC;AAEF,0FAA0F;AAC1F,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,OAAe;IAClD,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9E,IAAI,IAAI,GAA2E,IAAI,CAAC;IACxF,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACrD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACzF,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC;YAClF,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACzI,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC;QAClF,CAAC;IACL,CAAC;IACD,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,cAAc,KAAK,MAAM,CAAC,CAAC,CAAC,IAAA,+BAAiB,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAA,+BAAiB,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC5I,CAAC;AAEM,KAAK,UAAU,oBAAoB,CAAC,IAAgB;IACvD,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC5D,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,IAAI,CAAC,CAAC;QAC1D,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,SAAS,gBAAgB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5G,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAmB,gDAAgD,CAAC,EAAE,IAAI,CAAC;IACtG,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACtD,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CACpB;;iBAES,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE;;iBAEtD,EACT,KAAK,CACR,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,SAAS,2BAA2B,CAAC,CAAC;QACjE,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,iCAAiC,CAAC,CAAC;IACvE,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,2FAA2F,CAAC,CAAC;QACtG,GAAG,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IAC1F,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,IAAI,CAAC;gBACD,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACL,MAAM,GAAG,EAAE,CAAC;YAChB,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,KAAK,GAAG,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAA,sBAAc,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE9D,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,WAAW,MAAM,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC;QAC5E,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,cAAc,IAAI,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,iBAAiB,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7I,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,gBAAgB,IAAI,MAAM,EAAE,CAAC,CAAC;QAC1D,IAAI,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,QAAQ,CAAC,UAAU,eAAe,KAAK,CAAC,QAAQ,CAAC,eAAe,SAAS,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/H,CAAC;QAED,wFAAwF;QACxF,8CAA8C;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,CAAC;QACpD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAC9C,SAAS;YACb,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAG,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACnD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAClE,GAAG,CAAC,IAAI,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACvI,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,OAAO,CAAC,CAAC;AACb,CAAC"}

package/dist/cli/evaluate.d.ts

@@ -0,0 +1,15 @@
+/**
+ * `email-content evaluate` subcommand.
+ *
+ * Scores the current extractor against a stored reference set. The reference is re-scored from
+ * scratch every run, rather than reusing the baseline verdict captured at annotation time, so that
+ * a change to the rules is actually measured instead of being compared against its own past
+ * output.
+ *
+ * The reference here is LLM produced. It is treated as a reference, not as truth: two capable
+ * models agree with each other on about 80 percent of blocks, so a number in the high nineties
+ * would indicate a measurement problem, not a solved task.
+ */
+import { type ParsedArgs } from './args';
+export declare function evaluateCommand(args: ParsedArgs): Promise<number>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/cli/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH,OAAO,EAAoC,KAAK,UAAU,EAAE,MAAM,QAAQ,CAAC;AA8E3E,wBAAsB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAuJvE"}