@postalsys/email-content 0.0.1

package/dist/eval/metrics.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Evaluation metrics.
+ *
+ * Two rules shape this module.
+ *
+ * First, aggregate accuracy is close to meaningless here. The label distribution is skewed, so a
+ * classifier that guessed NEW_CONTENT for everything would score respectably while being useless.
+ * Per label precision and recall, and the catastrophic counts, are what matter.
+ *
+ * Second, the cost of errors is asymmetric by roughly ten to one. So the headline number is recall
+ * of genuine new content, measured in characters rather than blocks, because losing one long
+ * paragraph is worse than losing one short line.
+ */
+import { type Label } from '../types';
+export interface LabelledBlock {
+    path: string;
+    blockIndex: number;
+    charCount: number;
+    reference: Label;
+    predicted: Label;
+    /** Whether the prediction would actually be hidden from the reader. */
+    hidden: boolean;
+    /** Grouping key for per segment breakdowns, for example the client or the language. */
+    group?: string;
+}
+export interface LabelMetrics {
+    label: Label;
+    support: number;
+    predicted: number;
+    truePositives: number;
+    precision: number;
+    recall: number;
+    f1: number;
+}
+export interface RetentionMetrics {
+    /** Characters the reference calls authored by the sender. */
+    newContentChars: number;
+    /** How many of those remain visible. This is the headline number. */
+    newContentCharsRetained: number;
+    newContentCharRecall: number;
+    /** Blocks the reference calls authored, and how many remain visible. */
+    newContentBlocks: number;
+    newContentBlocksRetained: number;
+    newContentBlockRecall: number;
+    /** Of what was hidden, how much the reference agrees was history. */
+    hiddenChars: number;
+    hiddenCharsGenuinelyHistory: number;
+    hiddenPrecision: number;
+    /** History the reference identified that is still on screen. */
+    historyChars: number;
+    historyCharsStillVisible: number;
+    historyStillVisibleShare: number;
+}
+export interface CatastrophicCounts {
+    /** Messages where every authored block was hidden. */
+    allNewContentRemoved: number;
+    /** Messages where an authored block inside an interleaved reply was hidden. */
+    inlineAnswerRemoved: number;
+    /** Messages where quoted context was hidden while its answer stayed. */
+    inlineContextRemoved: number;
+    /** Messages where forwarded content was hidden. */
+    forwardedDiscarded: number;
+    /** Messages where the only visible content is a footer. */
+    onlyFooterRetained: number;
+    /** Messages producing nothing visible from a non empty body. */
+    emptyExtraction: number;
+    /** Messages where more than half the hidden characters were UNKNOWN in the reference. */
+    unknownHidden: number;
+    messages: number;
+}
+export interface EvaluationReport {
+    blocks: number;
+    messages: number;
+    exactAgreement: number;
+    perLabel: LabelMetrics[];
+    confusion: Map<string, number>;
+    retention: RetentionMetrics;
+    catastrophic: CatastrophicCounts;
+    byGroup: {
+        group: string;
+        blocks: number;
+        agreement: number;
+        newContentCharRecall: number;
+    }[];
+}
+export declare function evaluate(rows: readonly LabelledBlock[]): EvaluationReport;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/eval/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAU,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,KAAK,CAAC;IACjB,SAAS,EAAE,KAAK,CAAC;IACjB,uEAAuE;IACvE,MAAM,EAAE,OAAO,CAAC;IAChB,uFAAuF;IACvF,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IACzB,KAAK,EAAE,KAAK,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC7B,6DAA6D;IAC7D,eAAe,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wEAAwE;IACxE,gBAAgB,EAAE,MAAM,CAAC;IACzB,wBAAwB,EAAE,MAAM,CAAC;IACjC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,2BAA2B,EAAE,MAAM,CAAC;IACpC,eAAe,EAAE,MAAM,CAAC;IACxB,gEAAgE;IAChE,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IAC/B,sDAAsD;IACtD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wEAAwE;IACxE,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,eAAe,EAAE,MAAM,CAAC;IACxB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,kBAAkB,CAAC;IACjC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,oBAAoB,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACjG;AA2ID,wBAAgB,QAAQ,CAAC,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,gBAAgB,CAuCzE"}

package/dist/eval/metrics.js

@@ -0,0 +1,185 @@
+"use strict";
+/**
+ * Evaluation metrics.
+ *
+ * Two rules shape this module.
+ *
+ * First, aggregate accuracy is close to meaningless here. The label distribution is skewed, so a
+ * classifier that guessed NEW_CONTENT for everything would score respectably while being useless.
+ * Per label precision and recall, and the catastrophic counts, are what matter.
+ *
+ * Second, the cost of errors is asymmetric by roughly ten to one. So the headline number is recall
+ * of genuine new content, measured in characters rather than blocks, because losing one long
+ * paragraph is worse than losing one short line.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluate = evaluate;
+const types_1 = require("../types");
+function labelMetrics(rows, label) {
+    let truePositives = 0;
+    let support = 0;
+    let predicted = 0;
+    for (const row of rows) {
+        if (row.reference === label) {
+            support++;
+        }
+        if (row.predicted === label) {
+            predicted++;
+        }
+        if (row.reference === label && row.predicted === label) {
+            truePositives++;
+        }
+    }
+    const precision = predicted ? truePositives / predicted : 0;
+    const recall = support ? truePositives / support : 0;
+    return {
+        label,
+        support,
+        predicted,
+        truePositives,
+        precision,
+        recall,
+        f1: precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0
+    };
+}
+/** Labels the reference considers authored by the sender of this message. */
+const AUTHORED = new Set(['NEW_CONTENT', 'SIGNATURE']);
+function retention(rows) {
+    let newContentChars = 0;
+    let newContentCharsRetained = 0;
+    let newContentBlocks = 0;
+    let newContentBlocksRetained = 0;
+    let hiddenChars = 0;
+    let hiddenCharsGenuinelyHistory = 0;
+    let historyChars = 0;
+    let historyCharsStillVisible = 0;
+    for (const row of rows) {
+        if (row.reference === 'NEW_CONTENT') {
+            newContentChars += row.charCount;
+            newContentBlocks++;
+            if (!row.hidden) {
+                newContentCharsRetained += row.charCount;
+                newContentBlocksRetained++;
+            }
+        }
+        if (row.hidden) {
+            hiddenChars += row.charCount;
+            if (row.reference === 'REPLY_HISTORY') {
+                hiddenCharsGenuinelyHistory += row.charCount;
+            }
+        }
+        if (row.reference === 'REPLY_HISTORY') {
+            historyChars += row.charCount;
+            if (!row.hidden) {
+                historyCharsStillVisible += row.charCount;
+            }
+        }
+    }
+    return {
+        newContentChars,
+        newContentCharsRetained,
+        newContentCharRecall: newContentChars ? newContentCharsRetained / newContentChars : 1,
+        newContentBlocks,
+        newContentBlocksRetained,
+        newContentBlockRecall: newContentBlocks ? newContentBlocksRetained / newContentBlocks : 1,
+        hiddenChars,
+        hiddenCharsGenuinelyHistory,
+        hiddenPrecision: hiddenChars ? hiddenCharsGenuinelyHistory / hiddenChars : 1,
+        historyChars,
+        historyCharsStillVisible,
+        historyStillVisibleShare: historyChars ? historyCharsStillVisible / historyChars : 0
+    };
+}
+function catastrophic(rows) {
+    const byMessage = new Map();
+    for (const row of rows) {
+        const list = byMessage.get(row.path);
+        if (list) {
+            list.push(row);
+        }
+        else {
+            byMessage.set(row.path, [row]);
+        }
+    }
+    const counts = {
+        allNewContentRemoved: 0,
+        inlineAnswerRemoved: 0,
+        inlineContextRemoved: 0,
+        forwardedDiscarded: 0,
+        onlyFooterRetained: 0,
+        emptyExtraction: 0,
+        unknownHidden: 0,
+        messages: byMessage.size
+    };
+    for (const blocks of byMessage.values()) {
+        const authored = blocks.filter(block => block.reference === 'NEW_CONTENT');
+        const visible = blocks.filter(block => !block.hidden);
+        if (authored.length && authored.every(block => block.hidden)) {
+            counts.allNewContentRemoved++;
+        }
+        if (!visible.length && blocks.some(block => block.charCount > 0)) {
+            counts.emptyExtraction++;
+        }
+        if (visible.length && visible.every(block => AUTHORED.has(block.reference) && block.reference !== 'NEW_CONTENT')) {
+            counts.onlyFooterRetained++;
+        }
+        if (blocks.some(block => block.reference === 'FORWARDED_CONTENT' && block.hidden)) {
+            counts.forwardedDiscarded++;
+        }
+        if (blocks.some(block => block.reference === 'INLINE_QUOTE' && block.hidden)) {
+            // Hiding an inline quote leaves its answer without the question it answers.
+            counts.inlineContextRemoved++;
+        }
+        // An interleaved message is one with quoted context and authored replies alternating.
+        const interleaved = blocks.some(block => block.reference === 'INLINE_QUOTE') && blocks.some(block => block.reference === 'NEW_CONTENT');
+        if (interleaved && authored.some(block => block.hidden)) {
+            counts.inlineAnswerRemoved++;
+        }
+        const hiddenChars = blocks.filter(block => block.hidden).reduce((sum, block) => sum + block.charCount, 0);
+        const unknownHiddenChars = blocks.filter(block => block.hidden && block.reference === 'UNKNOWN').reduce((sum, block) => sum + block.charCount, 0);
+        if (hiddenChars > 0 && unknownHiddenChars / hiddenChars > 0.5) {
+            counts.unknownHidden++;
+        }
+    }
+    return counts;
+}
+function evaluate(rows) {
+    const confusion = new Map();
+    let agreed = 0;
+    for (const row of rows) {
+        const key = `${row.reference}|${row.predicted}`;
+        confusion.set(key, (confusion.get(key) ?? 0) + 1);
+        if (row.reference === row.predicted) {
+            agreed++;
+        }
+    }
+    const groups = new Map();
+    for (const row of rows) {
+        const key = row.group ?? 'all';
+        const list = groups.get(key);
+        if (list) {
+            list.push(row);
+        }
+        else {
+            groups.set(key, [row]);
+        }
+    }
+    return {
+        blocks: rows.length,
+        messages: new Set(rows.map(row => row.path)).size,
+        exactAgreement: rows.length ? agreed / rows.length : 0,
+        perLabel: types_1.LABELS.map(label => labelMetrics(rows, label)),
+        confusion,
+        retention: retention(rows),
+        catastrophic: catastrophic(rows),
+        byGroup: [...groups]
+            .map(([group, list]) => ({
+            group,
+            blocks: list.length,
+            agreement: list.filter(row => row.reference === row.predicted).length / list.length,
+            newContentCharRecall: retention(list).newContentCharRecall
+        }))
+            .sort((a, b) => b.blocks - a.blocks)
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/eval/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;AAoNH,4BAuCC;AAzPD,oCAA8C;AAyE9C,SAAS,YAAY,CAAC,IAA8B,EAAE,KAAY;IAC9D,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACd,CAAC;QACD,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC1B,SAAS,EAAE,CAAC;QAChB,CAAC;QACD,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YACrD,aAAa,EAAE,CAAC;QACpB,CAAC;IACL,CAAC;IACD,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO;QACH,KAAK;QACL,OAAO;QACP,SAAS;QACT,aAAa;QACb,SAAS;QACT,MAAM;QACN,EAAE,EAAE,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;KACnF,CAAC;AACN,CAAC;AAED,6EAA6E;AAC7E,MAAM,QAAQ,GAAuB,IAAI,GAAG,CAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC;AAElF,SAAS,SAAS,CAAC,IAA8B;IAC7C,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,wBAAwB,GAAG,CAAC,CAAC;IACjC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,2BAA2B,GAAG,CAAC,CAAC;IACpC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,wBAAwB,GAAG,CAAC,CAAC;IAEjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,GAAG,CAAC,SAAS,KAAK,aAAa,EAAE,CAAC;YAClC,eAAe,IAAI,GAAG,CAAC,SAAS,CAAC;YACjC,gBAAgB,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;gBACd,uBAAuB,IAAI,GAAG,CAAC,SAAS,CAAC;gBACzC,wBAAwB,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YACb,WAAW,IAAI,GAAG,CAAC,SAAS,CAAC;YAC7B,IAAI,GAAG,CAAC,SAAS,KAAK,eAAe,EAAE,CAAC;gBACpC,2BAA2B,IAAI,GAAG,CAAC,SAAS,CAAC;YACjD,CAAC;QACL,CAAC;QACD,IAAI,GAAG,CAAC,SAAS,KAAK,eAAe,EAAE,CAAC;YACpC,YAAY,IAAI,GAAG,CAAC,SAAS,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;gBACd,wBAAwB,IAAI,GAAG,CAAC,SAAS,CAAC;YAC9C,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO;QACH,eAAe;QACf,uBAAuB;QACvB,oBAAoB,EAAE,eAAe,CAAC,CAAC,CAAC,uBAAuB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QACrF,gBAAgB;QAChB,wBAAwB;QACxB,qBAAqB,EAAE,gBAAgB,CAAC,CAAC,CAAC,wBAAwB,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACzF,WAAW;QACX,2BAA2B;QAC3B,eAAe,EAAE,WAAW,CAAC,CAAC,CAAC,2BAA2B,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC5E,YAAY;QACZ,wBAAwB;QACxB,wBAAwB,EAAE,YAAY,CAAC,CAAC,CAAC,wBAAwB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;KACvF,CAAC;AACN,CAAC;AAED,SAAS,YAAY,CAAC,IAA8B;IAChD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAuB;QAC/B,oBAAoB,EAAE,CAAC;QACvB,mBAAmB,EAAE,CAAC;QACtB,oBAAoB,EAAE,CAAC;QACvB,kBAAkB,EAAE,CAAC;QACrB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,SAAS,CAAC,IAAI;KAC3B,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,aAAa,CAAC,CAAC;QAC3E,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtD,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;YAC/D,MAAM,CAAC,eAAe,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,KAAK,aAAa,CAAC,EAAE,CAAC;YAC/G,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAChF,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChC,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,cAAc,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,4EAA4E;YAC5E,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAClC,CAAC;QACD,sFAAsF;QACtF,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,aAAa,CAAC,CAAC;QACxI,IAAI,WAAW,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,mBAAmB,EAAE,CAAC;QACjC,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClJ,IAAI,WAAW,GAAG,CAAC,IAAI,kBAAkB,GAAG,WAAW,GAAG,GAAG,EAAE,CAAC;YAC5D,MAAM,CAAC,aAAa,EAAE,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAgB,QAAQ,CAAC,IAA8B;IACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QAChD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,KAAK,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,OAAO;QACH,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;QACjD,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,QAAQ,EAAE,cAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACxD,SAAS;QACT,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC;QAC1B,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;QAChC,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACrB,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YACnF,oBAAoB,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,oBAAoB;SAC7D,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;KAC3C,CAAC;AACN,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Public library API.
+ *
+ * `extract()` is the whole product surface: raw message in, classified segments out, with every
+ * segment traceable to a MIME part and a source range, and nothing ever discarded.
+ */
+import { type ParseOptions } from './mime/model';
+import { type ConfidenceConfig } from './classify/confidence';
+import { type HistoryDocument } from './match/history';
+import { type ExtractionResult, type Reason } from './types';
+export * from './types';
+export { parseMessage } from './mime/parse';
+export type { ParsedMessage, MimePart } from './mime/model';
+export type { Block, BlockFeatures, BlockKind, Evidence } from './segment/types';
+export { DEFAULT_CONFIDENCE } from './classify/confidence';
+export { buildHistoryDocument, matchBlocks, type HistoryDocument, type BlockMatch } from './match/history';
+export { corpusThreadResolver, folderThreadResolver, type ThreadResolver } from './corpus/thread-index';
+export { analyze, resolveHistory, senderTokensFrom, type MessageContent, type AnalyzeOptions, type AnalyzeResult } from './analyze';
+export interface ExtractOptions {
+    parse?: ParseOptions;
+    confidence?: ConfidenceConfig;
+    /**
+     * Earlier messages in the thread, most recent first. Optional: extraction works without any
+     * history, and history only ever adds evidence.
+     */
+    history?: readonly string[];
+    /** Pre-built history documents, for callers that resolve threads themselves. */
+    historyDocuments?: readonly HistoryDocument[];
+    /** Keep block level diagnostics on the result. Used by `extract --explain` and the review app. */
+    explain?: boolean;
+}
+export interface ExplainedResult extends ExtractionResult {
+    /** Present when `explain` is set. */
+    diagnostics?: {
+        structure: string;
+        primaryPart: string | null;
+        blocks: {
+            index: number;
+            kind: string;
+            label: string;
+            confidence: number;
+            reasons: Reason[];
+            evidence: {
+                id: string;
+                kind: string;
+                strength: number;
+            }[];
+            features: Record<string, unknown>;
+            preview: string;
+        }[];
+    };
+}
+export declare function extract(raw: Buffer | string, options?: ExtractOptions): Promise<ExplainedResult>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAiB,KAAK,YAAY,EAAsB,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,MAAM,EAAgB,MAAM,SAAS,CAAC;AAE9F,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAEpI,MAAM,WAAW,cAAc;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;OAGG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAC9C,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACrD,qCAAqC;IACrC,WAAW,CAAC,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,MAAM,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,EAAE,MAAM,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAC;gBAAC,QAAQ,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;YAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC;SACnB,EAAE,CAAC;KACP,CAAC;CACL;AAkFD,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,CA+C1G"}

package/dist/index.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Public library API.
+ *
+ * `extract()` is the whole product surface: raw message in, classified segments out, with every
+ * segment traceable to a MIME part and a source range, and nothing ever discarded.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.senderTokensFrom = exports.resolveHistory = exports.analyze = exports.folderThreadResolver = exports.corpusThreadResolver = exports.matchBlocks = exports.buildHistoryDocument = exports.DEFAULT_CONFIDENCE = exports.parseMessage = void 0;
+exports.extract = extract;
+const parse_1 = require("./mime/parse");
+const analyze_1 = require("./analyze");
+const types_1 = require("./types");
+__exportStar(require("./types"), exports);
+var parse_2 = require("./mime/parse");
+Object.defineProperty(exports, "parseMessage", { enumerable: true, get: function () { return parse_2.parseMessage; } });
+var confidence_1 = require("./classify/confidence");
+Object.defineProperty(exports, "DEFAULT_CONFIDENCE", { enumerable: true, get: function () { return confidence_1.DEFAULT_CONFIDENCE; } });
+var history_1 = require("./match/history");
+Object.defineProperty(exports, "buildHistoryDocument", { enumerable: true, get: function () { return history_1.buildHistoryDocument; } });
+Object.defineProperty(exports, "matchBlocks", { enumerable: true, get: function () { return history_1.matchBlocks; } });
+var thread_index_1 = require("./corpus/thread-index");
+Object.defineProperty(exports, "corpusThreadResolver", { enumerable: true, get: function () { return thread_index_1.corpusThreadResolver; } });
+Object.defineProperty(exports, "folderThreadResolver", { enumerable: true, get: function () { return thread_index_1.folderThreadResolver; } });
+var analyze_2 = require("./analyze");
+Object.defineProperty(exports, "analyze", { enumerable: true, get: function () { return analyze_2.analyze; } });
+Object.defineProperty(exports, "resolveHistory", { enumerable: true, get: function () { return analyze_2.resolveHistory; } });
+Object.defineProperty(exports, "senderTokensFrom", { enumerable: true, get: function () { return analyze_2.senderTokensFrom; } });
+/**
+ * Choose the text and HTML parts that hold the message body.
+ *
+ * The largest candidate of each type wins. A message can carry several text parts, for example a
+ * gateway appended disclaimer in `multipart/mixed(alternative(P,H), P)`, and the body is the large
+ * one. Attachments and parts inside a forwarded `message/rfc822` are excluded: an embedded message
+ * is content to be classified, not the container's own body.
+ */
+function selectBodyParts(message) {
+    const insideEmbedded = new Set();
+    for (const part of message.parts) {
+        if (!part.rfc822 && !part.embeddedRoot) {
+            continue;
+        }
+        for (const other of message.parts) {
+            let cursor = other;
+            while (cursor?.parentId) {
+                if (cursor.parentId === part.partId) {
+                    insideEmbedded.add(other.partId);
+                    break;
+                }
+                cursor = message.partsById.get(cursor.parentId);
+            }
+        }
+    }
+    let text = null;
+    let html = null;
+    for (const part of message.parts) {
+        if (part.disposition === 'attachment' || !part.decoded || insideEmbedded.has(part.partId)) {
+            continue;
+        }
+        if (part.contentType === 'text/plain') {
+            if (!text || part.decoded.text.length > (text.decoded?.text.length ?? 0)) {
+                text = part;
+            }
+        }
+        else if (part.contentType === 'text/html') {
+            if (!html || part.decoded.text.length > (html.decoded?.text.length ?? 0)) {
+                html = part;
+            }
+        }
+    }
+    return { text, html };
+}
+/**
+ * Extract the conversationally relevant content of a raw RFC 5322 message.
+ *
+ * Parses the MIME tree, picks the body parts, then hands off to analyze(). Delegating rather than
+ * duplicating the pipeline is deliberate: the two entry points must not be able to disagree about
+ * the same message, and a rule added to one must reach the other for free.
+ *
+ * Never throws for malformed input. A failure produces a result in which the whole body is a single
+ * visible UNKNOWN segment, plus a warning, because showing too much is always preferable to
+ * showing nothing.
+ */
+function degraded(text, warnings) {
+    return {
+        segments: text.trim()
+            ? [
+                {
+                    type: 'UNKNOWN',
+                    text,
+                    confidence: 0.5,
+                    visibility: 'show',
+                    reasons: ['degraded_to_unknown'],
+                    source: { mimePart: 'TEXT', representation: 'text', blockIndex: 0, start: 0, end: text.length }
+                }
+            ]
+            : [],
+        hiddenSegments: [],
+        overallConfidence: 0.5,
+        warnings,
+        originalAvailable: true,
+        primaryRepresentation: 'none',
+        extractorVersion: types_1.EXTRACTOR_VERSION,
+        historyUsed: false
+    };
+}
+async function extract(raw, options = {}) {
+    const buffer = typeof raw === 'string' ? Buffer.from(raw) : raw;
+    let message;
+    try {
+        message = await (0, parse_1.parseMessage)(buffer, options.parse);
+    }
+    catch (err) {
+        return degraded(buffer.toString('utf8'), [
+            { code: 'mime_parse_error', detail: err instanceof Error ? err.message.slice(0, 200) : 'unknown' },
+            { code: 'extraction_degraded' }
+        ]);
+    }
+    const parts = selectBodyParts(message);
+    const result = await (0, analyze_1.analyze)({
+        text: parts.text?.decoded?.text ?? null,
+        html: parts.html?.decoded?.text ?? null,
+        subject: message.headers.subject,
+        from: message.headers.from,
+        inReplyTo: message.headers.inReplyTo,
+        references: message.headers.references,
+        ...(parts.text ? { textPartId: parts.text.partId } : {}),
+        ...(parts.html ? { htmlPartId: parts.html.partId } : {})
+    }, {
+        ...(options.confidence ? { confidence: options.confidence } : {}),
+        ...(options.history ? { history: options.history } : {}),
+        ...(options.historyDocuments ? { historyDocuments: options.historyDocuments } : {}),
+        ...(options.explain ? { explain: true } : {})
+    });
+    // MIME level warnings belong to the message, not to the body analysis.
+    const warnings = [...message.warnings, ...result.warnings];
+    return {
+        segments: result.segments,
+        hiddenSegments: result.hiddenSegments,
+        overallConfidence: result.overallConfidence,
+        warnings,
+        originalAvailable: true,
+        primaryRepresentation: result.primaryRepresentation,
+        extractorVersion: result.extractorVersion,
+        historyUsed: result.historyUsed,
+        ...(result.diagnostics ? { diagnostics: { ...result.diagnostics, structure: message.structure } } : {})
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;AAkIH,0BA+CC;AA/KD,wCAA4C;AAI5C,uCAAoC;AACpC,mCAA8F;AAE9F,0CAAwB;AACxB,sCAA4C;AAAnC,qGAAA,YAAY,OAAA;AAGrB,oDAA2D;AAAlD,gHAAA,kBAAkB,OAAA;AAC3B,2CAA2G;AAAlG,+GAAA,oBAAoB,OAAA;AAAE,sGAAA,WAAW,OAAA;AAC1C,sDAAwG;AAA/F,oHAAA,oBAAoB,OAAA;AAAE,oHAAA,oBAAoB,OAAA;AACnD,qCAAoI;AAA3H,kGAAA,OAAO,OAAA;AAAE,yGAAA,cAAc,OAAA;AAAE,2GAAA,gBAAgB,OAAA;AAkClD;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,OAAsB;IAC3C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrC,SAAS;QACb,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,MAAM,GAAyB,KAAK,CAAC;YACzC,OAAO,MAAM,EAAE,QAAQ,EAAE,CAAC;gBACtB,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;oBAClC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjC,MAAM;gBACV,CAAC;gBACD,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACxF,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;gBACvE,IAAI,GAAG,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;gBACvE,IAAI,GAAG,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,QAAmB;IAC/C,OAAO;QACH,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE;YACjB,CAAC,CAAC;gBACI;oBACI,IAAI,EAAE,SAAS;oBACf,IAAI;oBACJ,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,MAAM;oBAClB,OAAO,EAAE,CAAC,qBAAqB,CAAC;oBAChC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE;iBAClG;aACJ;YACH,CAAC,CAAC,EAAE;QACR,cAAc,EAAE,EAAE;QAClB,iBAAiB,EAAE,GAAG;QACtB,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM;QAC7B,gBAAgB,EAAE,yBAAiB;QACnC,WAAW,EAAE,KAAK;KACrB,CAAC;AACN,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,GAAoB,EAAE,UAA0B,EAAE;IAC5E,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEhE,IAAI,OAAsB,CAAC;IAC3B,IAAI,CAAC;QACD,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACrC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE;YAClG,EAAE,IAAI,EAAE,qBAAqB,EAAE;SAClC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAO,EACxB;QACI,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACvC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACvC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO;QAChC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;QAC1B,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS;QACpC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;QACtC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,EACD;QACI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChD,CACJ,CAAC;IAEF,uEAAuE;IACvE,MAAM,QAAQ,GAAc,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEtE,OAAO;QACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;QACnD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1G,CAAC;AACN,CAAC"}

package/dist/match/history.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Stage 5: matching blocks against messages that already existed.
+ *
+ * This is the strongest evidence the pipeline can have. Every other signal is an inference from
+ * markup or wording: an attribution line suggests a quote follows, a blockquote suggests its
+ * contents are old. A match against the actual parent message is not an inference. The text
+ * demonstrably existed before this message was written, so it cannot be something the sender just
+ * wrote.
+ *
+ * In the working corpus 90.9 percent of replies resolve their In-Reply-To to a message that is also
+ * present, which makes this unusually available here. It remains strictly optional: extraction
+ * works with no history at all, and history only ever adds evidence.
+ *
+ * The matcher is deliberately conservative in one direction. A match can promote a block to
+ * REPLY_HISTORY, which hides it, so short blocks are never matched, and a high containment
+ * threshold is required. A non match never demotes anything.
+ */
+import { type Block } from '../segment/types';
+import { type ShingleIndex } from './shingle';
+export interface HistoryDocument {
+    /** Identifier for diagnostics, typically a path or Message-ID. */
+    id: string;
+    /** Normalised text of the whole message, all representations concatenated. */
+    normalised: string;
+    index: ShingleIndex;
+    /** True when this is the direct parent rather than an older ancestor. */
+    isParent: boolean;
+}
+export interface BlockMatch {
+    blockIndex: number;
+    documentId: string;
+    isParent: boolean;
+    containment: number;
+    /** True when the normalised text appeared verbatim. */
+    exact: boolean;
+}
+/**
+ * Build a searchable document from a raw earlier message.
+ *
+ * Both representations are folded into one text projection. Which representation the earlier
+ * message used has nothing to do with which one the reply quoted from, and clients routinely quote
+ * the HTML of a message into the plain text of a reply.
+ */
+export declare function buildHistoryDocument(raw: Buffer | string, id: string, isParent: boolean): Promise<HistoryDocument>;
+/**
+ * Build a searchable document from already parsed content.
+ *
+ * The counterpart to buildHistoryDocument for callers that never had a raw message. Both
+ * representations are folded together for the same reason: which one the earlier message used has
+ * nothing to do with which one the reply quoted from.
+ */
+export declare function buildHistoryDocumentFromContent(text: string, html: string, id: string, isParent: boolean): HistoryDocument;
+/**
+ * Match blocks against known history.
+ *
+ * Returns one match per block at most, preferring the direct parent over older ancestors, and
+ * higher containment over lower.
+ */
+export declare function matchBlocks(blocks: readonly Block[], history: readonly HistoryDocument[]): BlockMatch[];
+/** Convenience view for the classifier, which only needs to know which indices matched. */
+export declare function matchedIndices(matches: readonly BlockMatch[]): Set<number>;
+//# sourceMappingURL=history.d.ts.map

package/dist/match/history.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAsE,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAGlH,MAAM,WAAW,eAAe;IAC5B,kEAAkE;IAClE,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAkBxH;AAED;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,eAAe,CAU1H;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,eAAe,EAAE,GAAG,UAAU,EAAE,CA4CvG;AASD,2FAA2F;AAC3F,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAE1E"}