@postalsys/email-content 0.0.1

package/dist/cli/evaluate.js

@@ -0,0 +1,215 @@
+"use strict";
+/**
+ * `email-content evaluate` subcommand.
+ *
+ * Scores the current extractor against a stored reference set. The reference is re-scored from
+ * scratch every run, rather than reusing the baseline verdict captured at annotation time, so that
+ * a change to the rules is actually measured instead of being compared against its own past
+ * output.
+ *
+ * The reference here is LLM produced. It is treated as a reference, not as truth: two capable
+ * models agree with each other on about 80 percent of blocks, so a number in the high nineties
+ * would indicate a measurement problem, not a solved task.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateCommand = evaluateCommand;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const index_1 = require("../index");
+const store_1 = require("../corpus/store");
+const thread_index_1 = require("../corpus/thread-index");
+const metrics_1 = require("../eval/metrics");
+const parse_1 = require("../mime/parse");
+const types_1 = require("../types");
+const args_1 = require("./args");
+function pct(value) {
+    return `${(100 * value).toFixed(1)}%`;
+}
+function renderReport(report, title) {
+    const out = [];
+    out.push(title);
+    out.push(`  messages ${report.messages}, blocks ${report.blocks}, exact label agreement ${pct(report.exactAgreement)}`);
+    out.push('');
+    out.push('CONTENT RETENTION  (the headline numbers)');
+    const r = report.retention;
+    out.push(`  new content characters retained   ${pct(r.newContentCharRecall)}   (${r.newContentCharsRetained} of ${r.newContentChars})`);
+    out.push(`  new content blocks retained       ${pct(r.newContentBlockRecall)}   (${r.newContentBlocksRetained} of ${r.newContentBlocks})`);
+    out.push(`  hidden content genuinely history  ${pct(r.hiddenPrecision)}   (${r.hiddenCharsGenuinelyHistory} of ${r.hiddenChars})`);
+    out.push(`  history still visible             ${pct(r.historyStillVisibleShare)}   (${r.historyCharsStillVisible} of ${r.historyChars})`);
+    out.push('');
+    out.push('PER LABEL');
+    out.push('  label              support  predicted  precision  recall      f1');
+    for (const metric of report.perLabel) {
+        if (!metric.support && !metric.predicted) {
+            continue;
+        }
+        out.push(`  ${metric.label.padEnd(18)} ${String(metric.support).padStart(7)} ${String(metric.predicted).padStart(10)} ` +
+            `${pct(metric.precision).padStart(10)} ${pct(metric.recall).padStart(7)} ${pct(metric.f1).padStart(7)}`);
+    }
+    out.push('');
+    out.push('CONFUSION  (rows: reference, columns: predicted)');
+    const present = types_1.LABELS.filter(label => report.perLabel.find(metric => metric.label === label && (metric.support || metric.predicted)));
+    out.push(`  ${' '.repeat(18)}${present.map(label => label.slice(0, 8).padStart(9)).join('')}`);
+    for (const reference of present) {
+        const cells = present.map(predicted => String(report.confusion.get(`${reference}|${predicted}`) ?? 0).padStart(9)).join('');
+        out.push(`  ${reference.padEnd(18)}${cells}`);
+    }
+    out.push('');
+    out.push('CATASTROPHIC ERRORS  (counted per message, never averaged)');
+    const c = report.catastrophic;
+    const line = (label, value) => `  ${label.padEnd(42)} ${String(value).padStart(5)}  ${pct(value / Math.max(1, c.messages))}`;
+    out.push(line('all new content removed', c.allNewContentRemoved));
+    out.push(line('inline answer removed', c.inlineAnswerRemoved));
+    out.push(line('inline context removed', c.inlineContextRemoved));
+    out.push(line('forwarded content discarded', c.forwardedDiscarded));
+    out.push(line('only a footer retained', c.onlyFooterRetained));
+    out.push(line('empty extraction from non empty message', c.emptyExtraction));
+    out.push(line('mostly unknown content hidden', c.unknownHidden));
+    out.push('');
+    if (report.byGroup.length > 1) {
+        out.push('BY GROUP');
+        out.push('  group                     blocks  agreement  new content retained');
+        for (const group of report.byGroup.slice(0, 15)) {
+            out.push(`  ${group.group.padEnd(24)} ${String(group.blocks).padStart(6)} ${pct(group.agreement).padStart(10)} ${pct(group.newContentCharRecall).padStart(21)}`);
+        }
+        out.push('');
+    }
+    return out.join('\n');
+}
+async function evaluateCommand(args) {
+    const dbPath = (0, args_1.flagString)(args, 'db', 'data/corpus.sqlite');
+    if (!node_fs_1.default.existsSync(dbPath)) {
+        process.stderr.write(`no corpus database at ${dbPath}\n`);
+        return 2;
+    }
+    const model = (0, args_1.flagString)(args, 'model', 'gpt-5.6-sol');
+    const useHistory = (0, args_1.flagBool)(args, 'history', false);
+    const groupBy = (0, args_1.flagString)(args, 'group-by', 'client');
+    const limit = (0, args_1.flagNumber)(args, 'limit', 0);
+    const store = new store_1.CorpusStore(dbPath);
+    const root = store.queryOne('SELECT root FROM runs ORDER BY id DESC LIMIT 1')?.root;
+    if (!root) {
+        process.stderr.write('no completed scan run found\n');
+        store.close();
+        return 2;
+    }
+    // Latest annotation per (path, block) for the chosen model, so re-annotated messages do not
+    // count twice.
+    const reference = store.query(`SELECT a.path, a.block_index, a.label, a.char_count, m.client, m.lang, m.reply_style
+         FROM annotations a
+         JOIN annotation_runs r ON r.id = a.run_id
+         JOIN messages m ON m.path = a.path
+         WHERE r.model = ? AND a.source = 'llm'
+           AND a.id = (SELECT MAX(a2.id) FROM annotations a2
+                       JOIN annotation_runs r2 ON r2.id = a2.run_id
+                       WHERE a2.path = a.path AND a2.block_index = a.block_index AND r2.model = r.model)
+         ORDER BY a.path, a.block_index`, model);
+    if (!reference.length) {
+        process.stderr.write(`no annotations found for model ${model}. Run "annotate" first.\n`);
+        store.close();
+        return 2;
+    }
+    const byPath = new Map();
+    for (const row of reference) {
+        const list = byPath.get(row.path);
+        if (list) {
+            list.push(row);
+        }
+        else {
+            byPath.set(row.path, [row]);
+        }
+    }
+    const paths = [...byPath.keys()].slice(0, limit || undefined);
+    const resolver = useHistory ? (0, thread_index_1.corpusThreadResolver)(store, root) : null;
+    process.stderr.write(`evaluating ${paths.length} messages against ${model}${useHistory ? ', with thread history' : ''}\n`);
+    const rows = [];
+    let failures = 0;
+    let withHistory = 0;
+    for (const relPath of paths) {
+        const references = byPath.get(relPath);
+        let raw;
+        try {
+            raw = node_fs_1.default.readFileSync(node_path_1.default.join(root, relPath));
+        }
+        catch {
+            failures++;
+            continue;
+        }
+        let historyDocuments;
+        if (resolver) {
+            try {
+                const message = await (0, parse_1.parseMessage)(raw);
+                historyDocuments = await resolver.resolve(message.headers);
+                if (historyDocuments.length) {
+                    withHistory++;
+                }
+            }
+            catch {
+                historyDocuments = undefined;
+            }
+        }
+        let result;
+        try {
+            result = await (0, index_1.extract)(raw, historyDocuments ? { historyDocuments } : {});
+        }
+        catch {
+            failures++;
+            continue;
+        }
+        // Map every reference block to the prediction covering it. Segments merge adjacent blocks,
+        // so a block is matched by its index falling inside a segment's block range.
+        const predictions = new Map();
+        for (const segment of [...result.segments, ...result.hiddenSegments]) {
+            predictions.set(segment.source.blockIndex, { label: segment.type, hidden: segment.visibility === 'hide' });
+        }
+        // Segments start at a block index and run to the next one, so fill forward.
+        const ordered = [...predictions.keys()].sort((a, b) => a - b);
+        const lookup = (index) => {
+            let best = null;
+            for (const start of ordered) {
+                if (start <= index) {
+                    best = start;
+                }
+                else {
+                    break;
+                }
+            }
+            return best === null ? null : (predictions.get(best) ?? null);
+        };
+        const first = references[0];
+        const group = groupBy === 'lang' ? (first?.lang ?? 'unknown') : groupBy === 'reply-style' ? (first?.reply_style ?? 'unknown') : (first?.client ?? 'unknown');
+        for (const item of references) {
+            const prediction = lookup(item.block_index);
+            rows.push({
+                path: relPath,
+                blockIndex: item.block_index,
+                charCount: item.char_count,
+                reference: item.label,
+                // A block with no covering segment was dropped as empty, which is equivalent to
+                // being shown as nothing. Treat it as UNKNOWN and visible rather than silently
+                // excluding it from the score.
+                predicted: prediction?.label ?? 'UNKNOWN',
+                hidden: prediction?.hidden ?? false,
+                group
+            });
+        }
+    }
+    store.close();
+    if (failures) {
+        process.stderr.write(`${failures} message(s) could not be evaluated\n`);
+    }
+    if (resolver) {
+        process.stderr.write(`thread history available for ${withHistory} of ${paths.length} messages\n`);
+    }
+    const report = (0, metrics_1.evaluate)(rows);
+    if ((0, args_1.flagBool)(args, 'json', false)) {
+        process.stdout.write(`${JSON.stringify({ ...report, confusion: Object.fromEntries(report.confusion) }, null, 2)}\n`);
+        return 0;
+    }
+    process.stdout.write(`${renderReport(report, `EVALUATION vs ${model}${useHistory ? ' (with history)' : ' (no history)'}`)}\n`);
+    return 0;
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/cli/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;;;AAwFH,0CAuJC;AA7OD,sDAAyB;AACzB,0DAA6B;AAC7B,oCAAmC;AACnC,2CAA8C;AAC9C,yDAA8D;AAC9D,6CAAsF;AACtF,yCAA6C;AAC7C,oCAA8C;AAC9C,iCAA2E;AAY3E,SAAS,GAAG,CAAC,KAAa;IACtB,OAAO,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,YAAY,CAAC,MAAwB,EAAE,KAAa;IACzD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,GAAG,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,QAAQ,YAAY,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IACxH,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACtD,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;IAC3B,GAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,uBAAuB,OAAO,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC;IACxI,GAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,wBAAwB,OAAO,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC;IAC3I,GAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,2BAA2B,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;IACnI,GAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,CAAC,CAAC,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,wBAAwB,OAAO,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1I,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACtB,GAAG,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IAC/E,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACvC,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CACJ,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG;YAC1G,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAC9G,CAAC;IACN,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,cAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC/F,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5H,GAAG,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;IACvE,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9B,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;IAC7I,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAClE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC/D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IAC7E,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACrB,GAAG,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QAChF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CACJ,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CACzJ,CAAC;QACN,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAEM,KAAK,UAAU,eAAe,CAAC,IAAgB;IAClD,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC5D,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,IAAI,CAAC,CAAC;QAC1D,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,IAAA,eAAQ,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAA,iBAAU,EAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAE3C,MAAM,KAAK,GAAG,IAAI,mBAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAmB,gDAAgD,CAAC,EAAE,IAAI,CAAC;IACtG,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACtD,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACb,CAAC;IAED,4FAA4F;IAC5F,eAAe;IACf,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CACzB;;;;;;;;wCAQgC,EAChC,KAAK,CACR,CAAC;IAEF,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,KAAK,2BAA2B,CAAC,CAAC;QACzF,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,mCAAoB,EAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,qBAAqB,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAE3H,MAAM,IAAI,GAAoB,EAAE,CAAC;IACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAmB,CAAC;QACzD,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,EAAE,CAAC;YACX,SAAS;QACb,CAAC;QAED,IAAI,gBAAgB,CAAC;QACrB,IAAI,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,GAAG,CAAC,CAAC;gBACxC,gBAAgB,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;oBAC1B,WAAW,EAAE,CAAC;gBAClB,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACL,gBAAgB,GAAG,SAAS,CAAC;YACjC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACD,MAAM,GAAG,MAAM,IAAA,eAAO,EAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,EAAE,CAAC;YACX,SAAS;QACb,CAAC;QAED,2FAA2F;QAC3F,6EAA6E;QAC7E,MAAM,WAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;QACzE,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YACnE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC,CAAC;QAC/G,CAAC;QAED,4EAA4E;QAC5E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,CAAC,KAAa,EAA4C,EAAE;YACvE,IAAI,IAAI,GAAkB,IAAI,CAAC;YAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjB,IAAI,GAAG,KAAK,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACJ,MAAM;gBACV,CAAC;YACL,CAAC;YACD,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,KAAK,GACP,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,IAAI,SAAS,CAAC,CAAC;QAEnJ,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,CAAC;gBACN,IAAI,EAAE,OAAO;gBACb,UAAU,EAAE,IAAI,CAAC,WAAW;gBAC5B,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,SAAS,EAAE,IAAI,CAAC,KAAK;gBACrB,gFAAgF;gBAChF,+EAA+E;gBAC/E,+BAA+B;gBAC/B,SAAS,EAAE,UAAU,EAAE,KAAK,IAAI,SAAS;gBACzC,MAAM,EAAE,UAAU,EAAE,MAAM,IAAI,KAAK;gBACnC,KAAK;aACR,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,EAAE,CAAC;IAEd,IAAI,QAAQ,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,sCAAsC,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,QAAQ,EAAE,CAAC;QACX,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,WAAW,OAAO,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,MAAM,GAAG,IAAA,kBAAQ,EAAC,IAAI,CAAC,CAAC;IAE9B,IAAI,IAAA,eAAQ,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACrH,OAAO,CAAC,CAAC;IACb,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,iBAAiB,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IAC/H,OAAO,CAAC,CAAC;AACb,CAAC"}

package/dist/cli/extract.d.ts

@@ -0,0 +1,10 @@
+/**
+ * `email-content extract` subcommand.
+ *
+ * Human readable output shows the classification, the confidence and the reasons, so that a
+ * surprising decision can be understood without a debugger. Message content is printed here, which
+ * is fine and necessary: the operator asked for this specific message.
+ */
+import { type ParsedArgs } from './args';
+export declare function extractCommand(args: ParsedArgs): Promise<number>;
+//# sourceMappingURL=extract.d.ts.map

package/dist/cli/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/cli/extract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,EAA4C,KAAK,UAAU,EAAE,MAAM,QAAQ,CAAC;AAqBnF,wBAAsB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAiFtE"}

package/dist/cli/extract.js

@@ -0,0 +1,139 @@
+"use strict";
+/**
+ * `email-content extract` subcommand.
+ *
+ * Human readable output shows the classification, the confidence and the reasons, so that a
+ * surprising decision can be understood without a debugger. Message content is printed here, which
+ * is fine and necessary: the operator asked for this specific message.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractCommand = extractCommand;
+const node_fs_1 = __importDefault(require("node:fs"));
+const index_1 = require("../index");
+const args_1 = require("./args");
+const LABEL_WIDTH = 18;
+function truncate(text, limit) {
+    const collapsed = text.replace(/\s+/g, ' ').trim();
+    return collapsed.length > limit ? `${collapsed.slice(0, limit - 1)}…` : collapsed;
+}
+function renderSegment(segment, index, full) {
+    const head = `${String(index).padStart(2)}. ${segment.type.padEnd(LABEL_WIDTH)} ${segment.confidence.toFixed(2)}  ${segment.visibility.padEnd(8)} part ${segment.source.mimePart} [${segment.source.start}..${segment.source.end}]`;
+    const reasons = `    reasons: ${segment.reasons.join(', ') || 'none'}`;
+    const body = full
+        ? segment.text
+            .split('\n')
+            .map(line => `    | ${line}`)
+            .join('\n')
+        : `    | ${truncate(segment.text, 100)}`;
+    return `${head}\n${reasons}\n${body}`;
+}
+async function extractCommand(args) {
+    const file = args.positional[0];
+    if (!file) {
+        process.stderr.write('usage: email-content extract <message.eml> [--json] [--explain] [--full]\n');
+        return 2;
+    }
+    if (!node_fs_1.default.existsSync(file)) {
+        process.stderr.write(`file not found: ${file}\n`);
+        return 2;
+    }
+    const explain = (0, args_1.flagBool)(args, 'explain', false);
+    const raw = node_fs_1.default.readFileSync(file);
+    // Stage 5 evidence, when the caller can point at earlier messages. Either a corpus database,
+    // which resolves the thread by Message-ID, or a folder of .eml files to index.
+    let historyDocuments = [];
+    const historySource = (0, args_1.flagOptionalString)(args, 'history');
+    if (historySource) {
+        const { parseMessage } = await Promise.resolve().then(() => __importStar(require('../mime/parse')));
+        const message = await parseMessage(raw);
+        const isDatabase = /\.sqlite$/i.test(historySource);
+        if (isDatabase) {
+            const { CorpusStore } = await Promise.resolve().then(() => __importStar(require('../corpus/store')));
+            const { corpusThreadResolver } = await Promise.resolve().then(() => __importStar(require('../corpus/thread-index')));
+            const store = new CorpusStore(historySource);
+            const root = store.queryOne('SELECT root FROM runs ORDER BY id DESC LIMIT 1')?.root;
+            if (root) {
+                historyDocuments = await corpusThreadResolver(store, root).resolve(message.headers);
+            }
+            store.close();
+        }
+        else {
+            const { folderThreadResolver } = await Promise.resolve().then(() => __importStar(require('../corpus/thread-index')));
+            historyDocuments = await folderThreadResolver(historySource).resolve(message.headers);
+        }
+        process.stderr.write(`history: resolved ${historyDocuments.length} earlier message(s)\n`);
+    }
+    const result = await (0, index_1.extract)(raw, {
+        explain,
+        historyDocuments,
+        parse: { maxTextPartBytes: (0, args_1.flagNumber)(args, 'max-text-kb', 1024) * 1024 }
+    });
+    if ((0, args_1.flagBool)(args, 'json', false)) {
+        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        return 0;
+    }
+    const full = (0, args_1.flagBool)(args, 'full', false);
+    const out = [];
+    out.push(`primary representation: ${result.primaryRepresentation}`);
+    out.push(`overall confidence: ${result.overallConfidence.toFixed(3)}`);
+    out.push(`history used: ${result.historyUsed}`);
+    if (result.warnings.length) {
+        out.push(`warnings: ${result.warnings.map(warning => warning.code).join(', ')}`);
+    }
+    out.push('');
+    out.push(`VISIBLE (${result.segments.length})`);
+    result.segments.forEach((segment, index) => out.push(renderSegment(segment, index, full)));
+    if (result.hiddenSegments.length) {
+        out.push('', `HIDDEN (${result.hiddenSegments.length}, retained and recoverable)`);
+        result.hiddenSegments.forEach((segment, index) => out.push(renderSegment(segment, index, full)));
+    }
+    if (explain && result.diagnostics) {
+        out.push('', `DIAGNOSTICS  structure=${result.diagnostics.structure} part=${result.diagnostics.primaryPart}`);
+        for (const block of result.diagnostics.blocks) {
+            const evidence = block.evidence.map(item => `${item.id}@${item.strength}`).join(' ') || 'none';
+            out.push(`  #${String(block.index).padStart(2)} ${block.kind.padEnd(20)} -> ${block.label.padEnd(LABEL_WIDTH)} ${block.confidence.toFixed(2)}`);
+            out.push(`      evidence: ${evidence}`);
+            out.push(`      reasons:  ${block.reasons.join(', ')}`);
+            out.push(`      text:     ${truncate(block.preview, 100)}`);
+        }
+    }
+    process.stdout.write(`${out.join('\n')}\n`);
+    return 0;
+}
+//# sourceMappingURL=extract.js.map

package/dist/cli/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/cli/extract.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BH,wCAiFC;AA1GD,sDAAyB;AACzB,oCAAmC;AAGnC,iCAAmF;AAEnF,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,OAAO,SAAS,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AACtF,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,KAAa,EAAE,IAAa;IACjE,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;IACpO,MAAM,OAAO,GAAG,gBAAgB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,IAAI;QACb,CAAC,CAAC,OAAO,CAAC,IAAI;aACP,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;aAC5B,IAAI,CAAC,IAAI,CAAC;QACjB,CAAC,CAAC,SAAS,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;IAC7C,OAAO,GAAG,IAAI,KAAK,OAAO,KAAK,IAAI,EAAE,CAAC;AAC1C,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,IAAgB;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4EAA4E,CAAC,CAAC;QACnG,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC;QAClD,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,eAAQ,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAElC,6FAA6F;IAC7F,+EAA+E;IAC/E,IAAI,gBAAgB,GAAsB,EAAE,CAAC;IAC7C,MAAM,aAAa,GAAG,IAAA,yBAAkB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC1D,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,YAAY,EAAE,GAAG,wDAAa,eAAe,GAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACpD,IAAI,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,WAAW,EAAE,GAAG,wDAAa,iBAAiB,GAAC,CAAC;YACxD,MAAM,EAAE,oBAAoB,EAAE,GAAG,wDAAa,wBAAwB,GAAC,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,aAAa,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAmB,gDAAgD,CAAC,EAAE,IAAI,CAAC;YACtG,IAAI,IAAI,EAAE,CAAC;gBACP,gBAAgB,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxF,CAAC;YACD,KAAK,CAAC,KAAK,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,oBAAoB,EAAE,GAAG,wDAAa,wBAAwB,GAAC,CAAC;YACxE,gBAAgB,GAAG,MAAM,oBAAoB,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,gBAAgB,CAAC,MAAM,uBAAuB,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAA,eAAO,EAAC,GAAG,EAAE;QAC9B,OAAO;QACP,gBAAgB;QAChB,KAAK,EAAE,EAAE,gBAAgB,EAAE,IAAA,iBAAU,EAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,IAAI,EAAE;KAC5E,CAAC,CAAC;IAEH,IAAI,IAAA,eAAQ,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7D,OAAO,CAAC,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,IAAA,eAAQ,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,GAAG,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC;IACpE,GAAG,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvE,GAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3F,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,MAAM,CAAC,cAAc,CAAC,MAAM,6BAA6B,CAAC,CAAC;QACnF,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,OAAO,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,0BAA0B,MAAM,CAAC,WAAW,CAAC,SAAS,SAAS,MAAM,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9G,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;YAC/F,GAAG,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChJ,GAAG,CAAC,IAAI,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,CAAC;AACb,CAAC"}

package/dist/cli/main.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+/**
+ * email-content command line entry point.
+ *
+ * Commands are dispatched lazily so that, for example, `extract` does not pay for loading the
+ * corpus database layer.
+ */
+export {};
+//# sourceMappingURL=main.d.ts.map

package/dist/cli/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":";AACA;;;;;GAKG"}

package/dist/cli/main.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * email-content command line entry point.
+ *
+ * Commands are dispatched lazily so that, for example, `extract` does not pay for loading the
+ * corpus database layer.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const types_1 = require("../types");
+const args_1 = require("./args");
+const USAGE = `email-content <command> [options]
+
+Commands:
+  corpus scan <folder>        Scan a folder of .eml files into the corpus database
+    --db <path>               Database path (default data/corpus.sqlite)
+    --sample <n>              Deterministic sample of n files instead of the whole corpus
+    --seed <string>           Sampling seed (default "default")
+    --concurrency <n>         Worker threads (default: cores minus two)
+    --no-resume               Re-process files already recorded
+    --max-file-mb <n>         Skip files larger than this (default 32)
+    --timeout <ms>            Per file timeout (default 30000)
+
+  corpus stats                Print aggregate statistics
+    --db <path>               Database path
+    --json                    Emit JSON instead of Markdown
+    --out <path>              Write the Markdown report to a file
+
+  corpus threads              Summarise in-corpus thread parent resolution
+
+  extract <message.eml>       Classify one message into segments
+    --json                    Emit the full structured result
+    --explain                 Show per block evidence, features and reasons
+    --full                    Print complete segment text instead of a preview
+    --history <path>          Corpus .sqlite or folder of .eml files, to resolve thread history
+
+  benchmark                   Measure throughput and catastrophic error rates
+    --db <path>               Corpus database
+    --sample <n>              Messages to run (default 2000)
+    --traffic <class>         human, ticketing, mailing-list, bulk, all (default human)
+    --replies false           Include non replies
+
+  annotate                    Label blocks with an LLM instead of a human (sends content to OpenAI)
+    --model <id>              Model id (default gpt-5.6-sol)
+    --sample <n>              Messages to annotate (default 25)
+    --concurrency <n>         Parallel requests (default 4)
+    --traffic <class>         Traffic class to sample from (default human)
+    --seed <string>           Deterministic selection seed
+    --dry-run                 Count blocks and cost without calling the API
+    --no-resume               Re-annotate messages already done by this model
+
+  disagreements               Review blocks where annotator and baseline disagree (prints content)
+    --direction <d>           expensive, cheap, footers, all (default expensive)
+    --limit <n>               Rows to show (default 20)
+    --run <id>                Restrict to one annotation run
+
+  evaluate                    Score the extractor against an annotated reference set
+    --model <id>              Reference annotations from this model (default gpt-5.6-sol)
+    --history                 Resolve thread history from the corpus while evaluating
+    --group-by <field>        client, lang or reply-style (default client)
+    --json                    Emit the full report as JSON
+
+  version                     Print version information
+
+Privacy: no command prints message bodies, subjects or addresses.
+`;
+async function main(argv) {
+    const args = (0, args_1.parseArgs)(argv);
+    const command = args.positional[0];
+    switch (command) {
+        case 'corpus': {
+            const { corpusCommand } = await Promise.resolve().then(() => __importStar(require('./corpus')));
+            return corpusCommand(args.positional[1], { ...args, positional: args.positional.slice(2) });
+        }
+        case 'extract': {
+            const { extractCommand } = await Promise.resolve().then(() => __importStar(require('./extract')));
+            return extractCommand({ ...args, positional: args.positional.slice(1) });
+        }
+        case 'benchmark': {
+            const { benchmarkCommand } = await Promise.resolve().then(() => __importStar(require('./benchmark')));
+            return benchmarkCommand({ ...args, positional: args.positional.slice(1) });
+        }
+        case 'annotate': {
+            const { annotateCommand } = await Promise.resolve().then(() => __importStar(require('./annotate')));
+            return annotateCommand({ ...args, positional: args.positional.slice(1) });
+        }
+        case 'disagreements': {
+            const { disagreementsCommand } = await Promise.resolve().then(() => __importStar(require('./disagreements')));
+            return disagreementsCommand({ ...args, positional: args.positional.slice(1) });
+        }
+        case 'evaluate': {
+            const { evaluateCommand } = await Promise.resolve().then(() => __importStar(require('./evaluate')));
+            return evaluateCommand({ ...args, positional: args.positional.slice(1) });
+        }
+        case 'version':
+            process.stdout.write(`extractor ${types_1.EXTRACTOR_VERSION}, processing ${types_1.PROCESSING_VERSION}, node ${process.version}\n`);
+            return 0;
+        case 'help':
+        case undefined:
+            process.stdout.write(USAGE);
+            return command === undefined ? 2 : 0;
+        default:
+            process.stderr.write(`unknown command: ${command}\n\n${USAGE}`);
+            return 2;
+    }
+}
+main(process.argv.slice(2)).then(code => {
+    process.exitCode = code;
+}, (err) => {
+    const message = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`error: ${message}\n`);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=main.js.map

package/dist/cli/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":";;AACA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,oCAAiE;AACjE,iCAAmC;AAEnC,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsDb,CAAC;AAEF,KAAK,UAAU,IAAI,CAAC,IAAuB;IACvC,MAAM,IAAI,GAAG,IAAA,gBAAS,EAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAEnC,QAAQ,OAAO,EAAE,CAAC;QACd,KAAK,QAAQ,CAAC,CAAC,CAAC;YACZ,MAAM,EAAE,aAAa,EAAE,GAAG,wDAAa,UAAU,GAAC,CAAC;YACnD,OAAO,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,cAAc,EAAE,GAAG,wDAAa,WAAW,GAAC,CAAC;YACrD,OAAO,cAAc,CAAC,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,gBAAgB,EAAE,GAAG,wDAAa,aAAa,GAAC,CAAC;YACzD,OAAO,gBAAgB,CAAC,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,eAAe,EAAE,GAAG,wDAAa,YAAY,GAAC,CAAC;YACvD,OAAO,eAAe,CAAC,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,KAAK,eAAe,CAAC,CAAC,CAAC;YACnB,MAAM,EAAE,oBAAoB,EAAE,GAAG,wDAAa,iBAAiB,GAAC,CAAC;YACjE,OAAO,oBAAoB,CAAC,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YACd,MAAM,EAAE,eAAe,EAAE,GAAG,wDAAa,YAAY,GAAC,CAAC;YACvD,OAAO,eAAe,CAAC,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,KAAK,SAAS;YACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,yBAAiB,gBAAgB,0BAAkB,UAAU,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;YACpH,OAAO,CAAC,CAAC;QACb,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS;YACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC;YACI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC;YAChE,OAAO,CAAC,CAAC;IACjB,CAAC;AACL,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC5B,IAAI,CAAC,EAAE;IACH,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;AAC5B,CAAC,EACD,CAAC,GAAY,EAAE,EAAE;IACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACzB,CAAC,CACJ,CAAC"}

package/dist/corpus/analyze.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Corpus feature extraction.
+ *
+ * Turns a parsed message into the flat, privacy safe record stored in the corpus database. The hard
+ * rule here is that nothing derived from body text, subjects or addresses may leave this function
+ * in readable form. Subjects and addresses become salted hashes, sender domains are kept because
+ * they are needed for stratification, and body text is reduced to counts, ratios and a SimHash.
+ */
+import { type ReplyStyle } from '../canonical/text';
+import { type ClientId } from '../detect/client';
+import { type TrafficClass } from '../detect/traffic';
+import { type ParsedMessage } from '../mime/model';
+import { type LanguageCode } from '../util/lang';
+export type BodyKind = 'both' | 'plain-only' | 'html-only' | 'none';
+export interface MessageRecord {
+    messageId: string | null;
+    inReplyTo: string | null;
+    referencesCount: number;
+    referencesRoot: string | null;
+    dateMs: number | null;
+    fromDomain: string | null;
+    fromHash: string | null;
+    subjectHash: string | null;
+    subjectPrefix: 'reply' | 'forward' | 'plain' | 'none';
+    subjectLength: number;
+    structure: string;
+    bodyKind: BodyKind;
+    charsets: string;
+    encodings: string;
+    partCount: number;
+    attachmentCount: number;
+    hasRfc822: boolean;
+    textChars: number;
+    htmlChars: number;
+    flowed: boolean;
+    client: ClientId;
+    clientSource: string;
+    traffic: TrafficClass;
+    trafficSignals: string;
+    isReply: boolean;
+    isForward: boolean;
+    quoteDepthMax: number;
+    quotedRatio: number;
+    replyStyle: ReplyStyle | 'unknown';
+    /** Detector ids that fired, comma separated. Ids only, never matched text. */
+    markers: string;
+    htmlHints: string;
+    blockquoteCount: number;
+    tableCount: number;
+    hasImage: boolean;
+    suspectMalformedHtml: boolean;
+    lang: LanguageCode;
+    langScore: number;
+    bodySimhash: string;
+    warnings: string;
+    /** True when plain and HTML disagree about whether the message contains quoted history. */
+    quoteMarkingDisagrees: boolean;
+}
+/**
+ * Build the corpus record for one parsed message.
+ *
+ * @param message parsed MIME structure
+ * @param salt per database salt for the privacy preserving hashes
+ */
+export declare function analyzeMessage(message: ParsedMessage, salt: string): MessageRecord;
+//# sourceMappingURL=analyze.d.ts.map

package/dist/corpus/analyze.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../src/corpus/analyze.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAoC,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAqB,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEpE,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,EAAoB,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAEnE,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,YAAY,GAAG,WAAW,GAAG,MAAM,CAAC;AAEpE,MAAM,WAAW,aAAa;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,QAAQ,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,YAAY,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,UAAU,GAAG,SAAS,CAAC;IACnC,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,IAAI,EAAE,YAAY,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,2FAA2F;IAC3F,qBAAqB,EAAE,OAAO,CAAC;CAClC;AAoHD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CA6FlF"}