@postalsys/email-content 0.0.1

package/dist/canonical/text.js

@@ -0,0 +1,217 @@
+"use strict";
+/**
+ * Stage 2: plain text canonicalisation.
+ *
+ * Offsets in the rest of the pipeline are character offsets into the canonical string produced
+ * here, not byte offsets into the raw message. Traceability back to the file is via the owning MIME
+ * part's byte range. That split is deliberate: transfer decoding, charset conversion and line
+ * ending normalisation all change lengths, and maintaining a byte level offset map through three
+ * lossy transforms would be fragile for no practical gain.
+ *
+ * What this module does NOT do is unwrap hard wrapped lines. Unwrapping changes offsets and can
+ * merge a quoted line with a new one, which is exactly the mistake that loses content. Wrapping is
+ * recorded as a per line property and resolved later, in the matching projection only.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitQuotePrefix = splitQuotePrefix;
+exports.canonicaliseText = canonicaliseText;
+exports.profileQuoting = profileQuoting;
+/**
+ * Quote prefix matcher.
+ *
+ * Accepts the common forms: ">", "> ", ">>", "> > ", and up to three leading spaces before the
+ * first marker, which several clients add when re-wrapping. It deliberately does not accept "|"
+ * or ":" as quote markers: both appear far too often in ordinary text, tables and code, and a
+ * false positive here promotes new content into history.
+ */
+const QUOTE_PREFIX = /^[ \t]{0,3}((?:>[ \t]?)+)/;
+/** Split a line into its quote depth and remaining content. */
+function splitQuotePrefix(line) {
+    const match = QUOTE_PREFIX.exec(line);
+    if (!match || !match[1]) {
+        return { depth: 0, content: line };
+    }
+    const prefix = match[1];
+    let depth = 0;
+    for (let i = 0; i < prefix.length; i++) {
+        if (prefix[i] === '>') {
+            depth++;
+        }
+    }
+    return { depth, content: line.slice(match[0].length) };
+}
+/**
+ * Normalise decoded part text into the canonical form used by every later stage.
+ *
+ * Line endings become LF, a BOM is dropped, NUL bytes and lone carriage returns are removed, and
+ * the result is NFC normalised so that visually identical text compares equal during history
+ * matching. Nothing else is altered: no trimming, no whitespace collapsing, no unwrapping.
+ */
+function canonicaliseText(input) {
+    let text = input;
+    if (text.charCodeAt(0) === 0xfeff) {
+        text = text.slice(1);
+    }
+    text = text.replace(/\r\n?/g, '\n').replace(/\0/g, '');
+    try {
+        text = text.normalize('NFC');
+    }
+    catch {
+        // Extremely long or malformed input can defeat NFC. The unnormalised text is still usable.
+    }
+    const lines = [];
+    let offset = 0;
+    let index = 0;
+    while (offset <= text.length) {
+        let newline = text.indexOf('\n', offset);
+        if (newline === -1) {
+            newline = text.length;
+        }
+        const raw = text.slice(offset, newline);
+        const { depth, content } = splitQuotePrefix(raw);
+        lines.push({
+            index,
+            start: offset,
+            end: newline,
+            raw,
+            content,
+            depth,
+            blank: content.trim().length === 0
+        });
+        index++;
+        if (newline === text.length) {
+            break;
+        }
+        offset = newline + 1;
+    }
+    reattachWrapContinuations(lines);
+    return { text, lines };
+}
+/**
+ * Re-attach quoted lines whose wrapped continuation lost its quote prefix.
+ *
+ * Some clients hard wrap a quoted paragraph and fail to re-apply the ">" marker to the continuation:
+ *
+ *     > First of all, sorry for the delay in giving some final information
+ *     about the
+ *     > course. After carefully seeing your preferences, plus my
+ *
+ * "about the" is the tail of the quoted sentence, not something the sender wrote. Left alone it
+ * looks like unquoted content sitting between two quoted lines, which reads as an interleaved reply
+ * and makes the whole quoted message look like inline context that must stay visible. One message
+ * in the working corpus produced 30 such fragments and defeated the classifier completely.
+ *
+ * The test is deliberately narrow, because promoting real content into a quote is the expensive
+ * direction of error. All of these must hold: the line is short, it is immediately adjacent to
+ * quoted lines above and below with no blank line between, both neighbours are at the same depth,
+ * and the quoted line above ends mid sentence. A genuine inline reply is separated by a blank line
+ * and follows a quote that ended properly.
+ *
+ * Only the depth is adjusted. Offsets and raw text are untouched, so block tiling still reproduces
+ * the input exactly.
+ */
+function reattachWrapContinuations(lines) {
+    const MAX_CONTINUATION_CHARS = 60;
+    for (let i = 1; i < lines.length - 1; i++) {
+        const line = lines[i];
+        if (line.depth !== 0 || line.blank || line.raw.length > MAX_CONTINUATION_CHARS) {
+            continue;
+        }
+        const previous = lines[i - 1];
+        const next = lines[i + 1];
+        if (previous.depth === 0 || previous.blank || next.depth !== previous.depth || next.blank) {
+            continue;
+        }
+        // A quoted line that ended a sentence was not cut off mid flow, so what follows is new.
+        if (/[.!?:;)\]"']\s*$/.test(previous.content)) {
+            continue;
+        }
+        line.depth = previous.depth;
+        line.wrapContinuation = true;
+    }
+}
+/**
+ * Describe how quoting is laid out in a plain text body.
+ *
+ * The style categories are positional only. They describe where unquoted content sits relative to
+ * quoted content and make no claim about intent, because a trailing block after a quote is just as
+ * likely to be a signature as a genuine bottom posted answer. Deciding that is the classifier's
+ * job, not this function's.
+ */
+function profileQuoting(lines) {
+    const totalLines = lines.length;
+    let quotedLines = 0;
+    let maxDepth = 0;
+    let nonBlank = 0;
+    let firstQuote = -1;
+    let lastQuote = -1;
+    for (const line of lines) {
+        if (line.depth > 0) {
+            quotedLines++;
+            maxDepth = Math.max(maxDepth, line.depth);
+            if (firstQuote === -1) {
+                firstQuote = line.index;
+            }
+            lastQuote = line.index;
+        }
+        if (!line.blank) {
+            nonBlank++;
+        }
+    }
+    if (firstQuote === -1) {
+        return {
+            maxDepth: 0,
+            quotedLines: 0,
+            totalLines,
+            quotedRatio: 0,
+            style: 'none',
+            contentBefore: nonBlank,
+            contentBetween: 0,
+            contentAfter: 0
+        };
+    }
+    let contentBefore = 0;
+    let contentBetween = 0;
+    let contentAfter = 0;
+    for (const line of lines) {
+        if (line.depth > 0 || line.blank) {
+            continue;
+        }
+        if (line.index < firstQuote) {
+            contentBefore++;
+        }
+        else if (line.index > lastQuote) {
+            contentAfter++;
+        }
+        else {
+            contentBetween++;
+        }
+    }
+    let style;
+    if (contentBetween > 0) {
+        style = 'interleaved';
+    }
+    else if (contentBefore > 0 && contentAfter > 0) {
+        style = 'top-and-trailing';
+    }
+    else if (contentBefore > 0) {
+        style = 'top';
+    }
+    else if (contentAfter > 0) {
+        style = 'bottom';
+    }
+    else {
+        style = 'quote-only';
+    }
+    return {
+        maxDepth,
+        quotedLines,
+        totalLines,
+        quotedRatio: nonBlank ? quotedLines / nonBlank : 0,
+        style,
+        contentBefore,
+        contentBetween,
+        contentAfter
+    };
+}
+//# sourceMappingURL=text.js.map

package/dist/canonical/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/canonical/text.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;AAwDH,4CAaC;AASD,4CAwCC;AAuDD,wCA0EC;AA1MD;;;;;;;GAOG;AACH,MAAM,YAAY,GAAG,2BAA2B,CAAC;AAEjD,+DAA+D;AAC/D,SAAgB,gBAAgB,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACvC,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,KAAK,EAAE,CAAC;QACZ,CAAC;IACL,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,KAAa;IAC1C,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QAChC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACL,2FAA2F;IAC/F,CAAC;IAED,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC;YACP,KAAK;YACL,KAAK,EAAE,MAAM;YACb,GAAG,EAAE,OAAO;YACZ,GAAG;YACH,OAAO;YACP,KAAK;YACL,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;SACrC,CAAC,CAAC;QACH,KAAK,EAAE,CAAC;QACR,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM;QACV,CAAC;QACD,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,yBAAyB,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,SAAS,yBAAyB,CAAC,KAAiB;IAChD,MAAM,sBAAsB,GAAG,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAa,CAAC;QAClC,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;YAC7E,SAAS;QACb,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAa,CAAC;QAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAa,CAAC;QACtC,IAAI,QAAQ,CAAC,KAAK,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACxF,SAAS;QACb,CAAC;QACD,wFAAwF;QACxF,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,SAAS;QACb,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;IACjC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,KAA0B;IACrD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;YAC5B,CAAC;YACD,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,OAAO;YACH,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,CAAC;YACd,UAAU;YACV,WAAW,EAAE,CAAC;YACd,KAAK,EAAE,MAAM;YACb,aAAa,EAAE,QAAQ;YACvB,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC;SAClB,CAAC;IACN,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,GAAG,UAAU,EAAE,CAAC;YAC1B,aAAa,EAAE,CAAC;QACpB,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;YAChC,YAAY,EAAE,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,cAAc,EAAE,CAAC;QACrB,CAAC;IACL,CAAC;IAED,IAAI,KAAiB,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,GAAG,aAAa,CAAC;IAC1B,CAAC;SAAM,IAAI,aAAa,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QAC/C,KAAK,GAAG,kBAAkB,CAAC;IAC/B,CAAC;SAAM,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;SAAM,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,GAAG,QAAQ,CAAC;IACrB,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,YAAY,CAAC;IACzB,CAAC;IAED,OAAO;QACH,QAAQ;QACR,WAAW;QACX,UAAU;QACV,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClD,KAAK;QACL,aAAa;QACb,cAAc;QACd,YAAY;KACf,CAAC;AACN,CAAC"}

package/dist/classify/confidence.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Stage 8: confidence and visibility.
+ *
+ * All thresholds live here so that calibration against the labelled set can move them without
+ * touching detector or rule code. The numbers below are the deterministic baseline's starting
+ * point, chosen conservatively; Phase 6 replaces them with calibrated values.
+ */
+import { type Label, type Visibility } from '../types';
+export interface ConfidenceConfig {
+    /** At or above this, high confidence history may be hidden outright. */
+    hideThreshold: number;
+    /** At or above this, content is collapsed but one interaction away. */
+    collapseThreshold: number;
+    /** Minimum anchor strength before hiding is permitted at all. */
+    minAnchorStrength: number;
+}
+export declare const DEFAULT_CONFIDENCE: ConfidenceConfig;
+/**
+ * Map a label and confidence to a visibility band.
+ *
+ * The asymmetry is deliberate and is the core safety mechanism:
+ *   - only REPLY_HISTORY can ever be hidden outright
+ *   - FORWARDED_CONTENT is collapsed at most, because it was included on purpose
+ *   - INLINE_QUOTE stays visible, because it is the context an answer depends on
+ *   - UNKNOWN always stays visible
+ */
+export declare function visibilityFor(label: Label, confidence: number, config?: ConfidenceConfig): Visibility;
+/**
+ * Message level confidence.
+ *
+ * The weakest hide or collapse decision, not the average. An average would let a long run of easy
+ * blocks mask one risky decision, and the risky decision is the one that loses content.
+ */
+export declare function overallConfidence(decisions: readonly {
+    confidence: number;
+    visibility: Visibility;
+}[]): number;
+//# sourceMappingURL=confidence.d.ts.map

package/dist/classify/confidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"confidence.d.ts","sourceRoot":"","sources":["../../src/classify/confidence.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAEvD,MAAM,WAAW,gBAAgB;IAC7B,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iEAAiE;IACjE,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED,eAAO,MAAM,kBAAkB,EAAE,gBAIhC,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAE,gBAAqC,GAAG,UAAU,CAkBzH;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,UAAU,CAAA;CAAE,EAAE,GAAG,MAAM,CAM9G"}

package/dist/classify/confidence.js

@@ -0,0 +1,59 @@
+"use strict";
+/**
+ * Stage 8: confidence and visibility.
+ *
+ * All thresholds live here so that calibration against the labelled set can move them without
+ * touching detector or rule code. The numbers below are the deterministic baseline's starting
+ * point, chosen conservatively; Phase 6 replaces them with calibrated values.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_CONFIDENCE = void 0;
+exports.visibilityFor = visibilityFor;
+exports.overallConfidence = overallConfidence;
+exports.DEFAULT_CONFIDENCE = {
+    hideThreshold: 0.95,
+    collapseThreshold: 0.75,
+    minAnchorStrength: 0.85
+};
+/**
+ * Map a label and confidence to a visibility band.
+ *
+ * The asymmetry is deliberate and is the core safety mechanism:
+ *   - only REPLY_HISTORY can ever be hidden outright
+ *   - FORWARDED_CONTENT is collapsed at most, because it was included on purpose
+ *   - INLINE_QUOTE stays visible, because it is the context an answer depends on
+ *   - UNKNOWN always stays visible
+ */
+function visibilityFor(label, confidence, config = exports.DEFAULT_CONFIDENCE) {
+    switch (label) {
+        case 'REPLY_HISTORY':
+            if (confidence >= config.hideThreshold) {
+                return 'hide';
+            }
+            return confidence >= config.collapseThreshold ? 'collapse' : 'show';
+        case 'FORWARDED_CONTENT':
+            return confidence >= config.collapseThreshold ? 'collapse' : 'show';
+        case 'SIGNATURE':
+        case 'DISCLAIMER':
+            return confidence >= config.collapseThreshold ? 'collapse' : 'show';
+        case 'INLINE_QUOTE':
+        case 'NEW_CONTENT':
+        case 'UNKNOWN':
+        default:
+            return 'show';
+    }
+}
+/**
+ * Message level confidence.
+ *
+ * The weakest hide or collapse decision, not the average. An average would let a long run of easy
+ * blocks mask one risky decision, and the risky decision is the one that loses content.
+ */
+function overallConfidence(decisions) {
+    const risky = decisions.filter(decision => decision.visibility !== 'show');
+    if (!risky.length) {
+        return 1;
+    }
+    return Math.min(...risky.map(decision => decision.confidence));
+}
+//# sourceMappingURL=confidence.js.map

package/dist/classify/confidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"confidence.js","sourceRoot":"","sources":["../../src/classify/confidence.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;AA4BH,sCAkBC;AAQD,8CAMC;AA/CY,QAAA,kBAAkB,GAAqB;IAChD,aAAa,EAAE,IAAI;IACnB,iBAAiB,EAAE,IAAI;IACvB,iBAAiB,EAAE,IAAI;CAC1B,CAAC;AAEF;;;;;;;;GAQG;AACH,SAAgB,aAAa,CAAC,KAAY,EAAE,UAAkB,EAAE,SAA2B,0BAAkB;IACzG,QAAQ,KAAK,EAAE,CAAC;QACZ,KAAK,eAAe;YAChB,IAAI,UAAU,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBACrC,OAAO,MAAM,CAAC;YAClB,CAAC;YACD,OAAO,UAAU,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,KAAK,mBAAmB;YACpB,OAAO,UAAU,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,KAAK,WAAW,CAAC;QACjB,KAAK,YAAY;YACb,OAAO,UAAU,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QACxE,KAAK,cAAc,CAAC;QACpB,KAAK,aAAa,CAAC;QACnB,KAAK,SAAS,CAAC;QACf;YACI,OAAO,MAAM,CAAC;IACtB,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,SAAoE;IAClG,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;IAC3E,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC;IACb,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;AACnE,CAAC"}

package/dist/classify/reconcile.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Stage 7: plain text and HTML reconciliation.
+ *
+ * Measured over the full corpus, 79.2 percent of messages carry both representations, and among
+ * human replies that carry both, a quarter disagree about whether the message contains quoted
+ * history at all. The disagreement is almost always in the same direction: the HTML part has a
+ * quote container and the plain part is a flattened render where the quote markers are simply gone.
+ *
+ * So HTML is the primary representation when both exist and the HTML actually carries structure.
+ * The plain part is still classified independently, and its verdict is used two ways: to confirm,
+ * which raises nothing but costs nothing, and to contradict, which caps confidence so that a
+ * disputed region is collapsed rather than hidden.
+ */
+import { type Representation } from '../types';
+import { type BlockDecision } from './rules';
+export interface RepresentationResult {
+    representation: Representation;
+    decisions: BlockDecision[];
+    /** Total characters of authored content, used to spot a stub representation. */
+    contentChars: number;
+    /** True when the representation contains at least one structural anchor. */
+    hasAnchor: boolean;
+    /** Blocks matched against known history, if any was supplied. */
+    matches?: readonly {
+        blockIndex: number;
+        containment: number;
+        exact: boolean;
+        isParent: boolean;
+    }[];
+}
+export interface ReconcileOutcome {
+    primary: RepresentationResult;
+    secondary: RepresentationResult | null;
+    disagrees: boolean;
+}
+/**
+ * Choose which representation drives the output.
+ *
+ * HTML wins by default when both exist, because it is where the structure survives. Plain text
+ * wins when the HTML is a stub, which happens with clients that send a token HTML wrapper around a
+ * text body, and when only the plain part has an anchor.
+ */
+export declare function pickPrimary(text: RepresentationResult | null, html: RepresentationResult | null): ReconcileOutcome;
+/**
+ * Apply cross representation evidence to the primary decisions.
+ *
+ * The only adjustment made is downward. Raising confidence because the other representation agrees
+ * would let two copies of the same mistake reinforce each other, and the cost of a wrong hide is
+ * much higher than the cost of a collapse that could have been a hide.
+ */
+export declare function reconcile(outcome: ReconcileOutcome, collapseCeiling?: number): BlockDecision[];
+//# sourceMappingURL=reconcile.d.ts.map

package/dist/classify/reconcile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../src/classify/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,UAAU,CAAC;AAC5D,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,MAAM,WAAW,oBAAoB;IACjC,cAAc,EAAE,cAAc,CAAC;IAC/B,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,gFAAgF;IAChF,YAAY,EAAE,MAAM,CAAC;IACrB,4EAA4E;IAC5E,SAAS,EAAE,OAAO,CAAC;IACnB,iEAAiE;IACjE,OAAO,CAAC,EAAE,SAAS;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACvG;AAED,MAAM,WAAW,gBAAgB;IAC7B,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACvC,SAAS,EAAE,OAAO,CAAC;CACtB;AAgBD;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,oBAAoB,GAAG,IAAI,EAAE,IAAI,EAAE,oBAAoB,GAAG,IAAI,GAAG,gBAAgB,CA8BlH;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,SAAM,GAAG,aAAa,EAAE,CA8B3F"}

package/dist/classify/reconcile.js

@@ -0,0 +1,100 @@
+"use strict";
+/**
+ * Stage 7: plain text and HTML reconciliation.
+ *
+ * Measured over the full corpus, 79.2 percent of messages carry both representations, and among
+ * human replies that carry both, a quarter disagree about whether the message contains quoted
+ * history at all. The disagreement is almost always in the same direction: the HTML part has a
+ * quote container and the plain part is a flattened render where the quote markers are simply gone.
+ *
+ * So HTML is the primary representation when both exist and the HTML actually carries structure.
+ * The plain part is still classified independently, and its verdict is used two ways: to confirm,
+ * which raises nothing but costs nothing, and to contradict, which caps confidence so that a
+ * disputed region is collapsed rather than hidden.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pickPrimary = pickPrimary;
+exports.reconcile = reconcile;
+/** Fraction of characters classified as hidden or collapsed history. */
+function historyShare(result) {
+    let history = 0;
+    let total = 0;
+    for (const decision of result.decisions) {
+        const chars = decision.block.content.length;
+        total += chars;
+        if (decision.label === 'REPLY_HISTORY') {
+            history += chars;
+        }
+    }
+    return total > 0 ? history / total : 0;
+}
+/**
+ * Choose which representation drives the output.
+ *
+ * HTML wins by default when both exist, because it is where the structure survives. Plain text
+ * wins when the HTML is a stub, which happens with clients that send a token HTML wrapper around a
+ * text body, and when only the plain part has an anchor.
+ */
+function pickPrimary(text, html) {
+    if (text && !html) {
+        return { primary: text, secondary: null, disagrees: false };
+    }
+    if (html && !text) {
+        return { primary: html, secondary: null, disagrees: false };
+    }
+    if (!text || !html) {
+        throw new Error('reconcile called with no representations');
+    }
+    // A representation with almost no content is a stub, not a real alternative.
+    const htmlIsStub = html.contentChars * 4 < text.contentChars;
+    const textIsStub = text.contentChars * 4 < html.contentChars;
+    let primary = html;
+    let secondary = text;
+    if (htmlIsStub && !textIsStub) {
+        primary = text;
+        secondary = html;
+    }
+    else if (!html.hasAnchor && text.hasAnchor && !textIsStub) {
+        primary = text;
+        secondary = html;
+    }
+    // Disagreement is measured on the share of content treated as history, not on block counts,
+    // because the two representations rarely segment into the same number of blocks.
+    const disagrees = Math.abs(historyShare(primary) - historyShare(secondary)) > 0.25;
+    return { primary, secondary, disagrees };
+}
+/**
+ * Apply cross representation evidence to the primary decisions.
+ *
+ * The only adjustment made is downward. Raising confidence because the other representation agrees
+ * would let two copies of the same mistake reinforce each other, and the cost of a wrong hide is
+ * much higher than the cost of a collapse that could have been a hide.
+ */
+function reconcile(outcome, collapseCeiling = 0.9) {
+    const { primary, secondary, disagrees } = outcome;
+    const reason = primary.representation === 'html' ? 'html_primary_representation' : 'text_primary_representation';
+    for (const decision of primary.decisions) {
+        if (!decision.reasons.includes(reason)) {
+            decision.reasons.push(reason);
+        }
+    }
+    if (!secondary || !disagrees) {
+        return primary.decisions;
+    }
+    for (const decision of primary.decisions) {
+        if (decision.label !== 'REPLY_HISTORY') {
+            continue;
+        }
+        if (decision.reasons.includes('matched_parent_message')) {
+            // History matching is evidence about the world, not about markup. A disagreement
+            // between two renderings of the same message does not weaken it.
+            continue;
+        }
+        decision.confidence = Math.min(decision.confidence, collapseCeiling);
+        if (!decision.reasons.includes('representations_disagree')) {
+            decision.reasons.push('representations_disagree');
+        }
+    }
+    return primary.decisions;
+}
+//# sourceMappingURL=reconcile.js.map

package/dist/classify/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../../src/classify/reconcile.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;AA2CH,kCA8BC;AASD,8BA8BC;AA1FD,wEAAwE;AACxE,SAAS,YAAY,CAAC,MAA4B;IAC9C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAC5C,KAAK,IAAI,KAAK,CAAC;QACf,IAAI,QAAQ,CAAC,KAAK,KAAK,eAAe,EAAE,CAAC;YACrC,OAAO,IAAI,KAAK,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,IAAiC,EAAE,IAAiC;IAC5F,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAChE,CAAC;IACD,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IAED,6EAA6E;IAC7E,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;IAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;IAE7D,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,UAAU,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,OAAO,GAAG,IAAI,CAAC;QACf,SAAS,GAAG,IAAI,CAAC;IACrB,CAAC;SAAM,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1D,OAAO,GAAG,IAAI,CAAC;QACf,SAAS,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,4FAA4F;IAC5F,iFAAiF;IACjF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC;IAEnF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,SAAS,CAAC,OAAyB,EAAE,eAAe,GAAG,GAAG;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,MAAM,GAAW,OAAO,CAAC,cAAc,KAAK,MAAM,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,6BAA6B,CAAC;IAEzH,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,SAAS,CAAC;IAC7B,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,KAAK,KAAK,eAAe,EAAE,CAAC;YACrC,SAAS;QACb,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;YACtD,iFAAiF;YACjF,iEAAiE;YACjE,SAAS;QACb,CAAC;QACD,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QACrE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;YACzD,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACtD,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,SAAS,CAAC;AAC7B,CAAC"}

package/dist/classify/rules.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Stage 6: rule based sequence classification.
+ *
+ * Blocks are classified in context, never independently. The rules encode the four sequences that
+ * matter, in the order they are resolved:
+ *
+ *   NEW_CONTENT, SIGNATURE, REPLY_HEADER, REPLY_HISTORY          top posting, the common case
+ *   INLINE_QUOTE, NEW_CONTENT, INLINE_QUOTE, NEW_CONTENT         interleaved answers
+ *   NEW_CONTENT, FORWARD_HEADER, FORWARDED_CONTENT               forwarding with a comment
+ *   REPLY_HISTORY, NEW_CONTENT                                   bottom posting
+ *
+ * The load bearing rule is the anchor requirement. A quote region may only become REPLY_HISTORY
+ * when something independent of the quoting itself says that it is history: an attribution line, a
+ * client quote container, or a match against a message that already existed. Quote depth alone
+ * gives INLINE_QUOTE or UNKNOWN, both of which stay visible.
+ *
+ * The reason a bare quote is not treated as history is the difference between the two examples in
+ * the brief. Quoted questions with answers under them are context the reader needs. A trailing
+ * quoted copy of the previous mail is noise. Only the surrounding structure tells them apart.
+ */
+import { type Label, type Reason } from '../types';
+import { type Block } from '../segment/types';
+import { type ConfidenceConfig } from './confidence';
+export interface BlockDecision {
+    block: Block;
+    label: Label;
+    confidence: number;
+    reasons: Reason[];
+}
+export interface ClassifyOptions {
+    config?: ConfidenceConfig;
+    /**
+     * Indices of blocks whose text was found in a known earlier message. Supplies the anchor for
+     * blocks that have no structural evidence of their own.
+     */
+    historyMatchedIndices?: ReadonlySet<number>;
+    /**
+     * Display name and local part of the sender, lowercased. A trailing short block containing the
+     * sender's own name is strong evidence of a signature: it is how nearly every signature begins,
+     * and unlike a greeting phrase it is language independent.
+     */
+    senderTokens?: readonly string[];
+}
+/**
+ * Classify one representation's blocks.
+ *
+ * Runs in four passes so that later passes can see the whole picture: footers first, because they
+ * bound where history can start; then anchors; then regions; then defaults.
+ */
+export declare function classifyBlocks(blocks: readonly Block[], options?: ClassifyOptions): BlockDecision[];
+//# sourceMappingURL=rules.d.ts.map

package/dist/classify/rules.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rules.d.ts","sourceRoot":"","sources":["../../src/classify/rules.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAsB,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEzE,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC5C;;;;OAIG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAqED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,GAAE,eAAoB,GAAG,aAAa,EAAE,CA4lBvG"}