@postalsys/email-content 0.0.5 → 0.1.1

package/dist/eval/oracle.js

@@ -0,0 +1,250 @@
+"use strict";
+/**
+ * Independent safety oracles for the structural benchmark.
+ *
+ * The benchmark exists to detect the failure this project is built to prevent: a message that had
+ * content going in and has none coming out. It cannot do that by asking the extractor, and the
+ * previous version did exactly that:
+ *
+ *     const hadContent = visibleChars + hiddenChars > 0;
+ *
+ * That reads "did the input have content" off the output. When extraction dropped everything the
+ * flag went false, the catastrophic counter did not increment, and the benchmark reported zero
+ * losses while a reproducible total loss case existed.
+ *
+ * So the code here shares the *definition* of a body with the extractor, and shares none of its
+ * *implementation*. It re-parses the message, walks the flat part list itself, and never calls the
+ * selection logic it is meant to check. Re-parsing costs a second MIME pass per message, which is
+ * the right trade for a release gate.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.embeddedPartIds = embeddedPartIds;
+exports.eligibleBodyParts = eligibleBodyParts;
+exports.inventoryVisibleInput = inventoryVisibleInput;
+exports.requiredCoverageGroups = requiredCoverageGroups;
+exports.coverageOf = coverageOf;
+const text_1 = require("../canonical/text");
+const html_1 = require("../canonical/html");
+const normalize_1 = require("../match/normalize");
+const shingle_1 = require("../match/shingle");
+/** Part ids that sit inside an embedded `message/rfc822`, which is content and not the outer body. */
+function embeddedPartIds(message) {
+    const containers = message.parts.filter(part => part.rfc822 || part.embeddedRoot);
+    const inside = new Set();
+    if (!containers.length) {
+        return inside;
+    }
+    for (const part of message.parts) {
+        let cursor = part;
+        while (cursor?.parentId) {
+            if (containers.some(container => container.partId === cursor?.parentId)) {
+                inside.add(part.partId);
+                break;
+            }
+            cursor = message.partsById.get(cursor.parentId);
+        }
+    }
+    return inside;
+}
+/** A decoded body-bearing leaf, by the written MIME rules rather than by the selection heuristic. */
+function eligibleBodyParts(message) {
+    const embedded = embeddedPartIds(message);
+    return message.parts.filter(part => (part.contentType === 'text/plain' || part.contentType === 'text/html') &&
+        part.disposition !== 'attachment' &&
+        Boolean(part.decoded) &&
+        !embedded.has(part.partId));
+}
+/**
+ * Count plausible visible body text in a parsed message, before extraction runs.
+ *
+ * Deliberately over-inclusive: it counts every eligible part, including both members of a
+ * `multipart/alternative`, because its only question is whether the message had anything visible to
+ * show. It is not, and must not be used as, a demand that every part appear in the output.
+ *
+ * Script, style, comment and markup text is excluded, so an image-only newsletter or a body that is
+ * nothing but a tracking pixel does not count as visible text that extraction then lost.
+ */
+function inventoryVisibleInput(message) {
+    let visibleChars = 0;
+    const parts = eligibleBodyParts(message);
+    for (const part of parts) {
+        visibleChars += visibleTextOf(part).length;
+    }
+    return { visibleChars, eligibleParts: parts.length, hasVisibleText: visibleChars > 0 };
+}
+/**
+ * Text a reader would actually have seen in one part.
+ *
+ * Quote prefixes are stripped rather than counted. A plain part whose entire body is `>` on three
+ * lines contains no text at all, and counting the markers made the gate report a total content loss
+ * on a message that had nothing to lose. Script, style, comment and markup text is excluded the
+ * same way, so an image-only body does not look like text that extraction dropped.
+ */
+function visibleTextOf(part) {
+    const raw = part.decoded?.text ?? '';
+    if (!raw) {
+        return '';
+    }
+    if (part.contentType === 'text/html') {
+        return (0, html_1.stripTagsToText)(raw).trim();
+    }
+    return (0, text_1.canonicaliseText)(raw)
+        .lines.filter(line => !line.blank)
+        .map(line => line.content)
+        .join('\n')
+        .trim();
+}
+/**
+ * Supported body leaf, by the same written rules the plan uses, deliberately reimplemented here.
+ *
+ * A part with no visible text is not required coverage. An image-only newsletter and a body that is
+ * nothing but a tracking pixel both have a body part and nothing for the output to carry, and
+ * demanding a segment for them would report "content lost" on messages where none existed.
+ */
+function isSupportedLeaf(part) {
+    return ((part.contentType === 'text/plain' || part.contentType === 'text/html') &&
+        part.disposition !== 'attachment' &&
+        Boolean(part.decoded) &&
+        visibleTextOf(part) !== '');
+}
+function supportedLeavesUnder(message, part, into, depth = 0) {
+    if (depth > 32 || part.disposition === 'attachment' || part.rfc822 || part.embeddedRoot) {
+        return;
+    }
+    if (isSupportedLeaf(part)) {
+        into.push(part.partId);
+        return;
+    }
+    for (const childId of part.childIds) {
+        const child = message.partsById.get(childId);
+        if (child) {
+            supportedLeavesUnder(message, child, into, depth + 1);
+        }
+    }
+}
+function bareContentId(value) {
+    return (value ?? '').trim().replace(/^</, '').replace(/>$/, '').toLowerCase();
+}
+/**
+ * Fold a `mixed` container's misencoded alternatives into single equivalence groups.
+ *
+ * Some clients use `multipart/mixed` where they mean `multipart/alternative`, `mixed(H,P)` most
+ * often. The extractor merges such a pair into one section when their content matches, so only one
+ * of the two part ids reaches the output, and a per-part rule would report the other as lost
+ * content when nothing was lost.
+ *
+ * Decided by content rather than structure, exactly as the selection code decides it, and
+ * implemented separately here on purpose. An oracle that called the selection code would agree with
+ * it by construction and could never catch it being wrong. These two must be reviewed together
+ * against `docs/adr-001-document-global-classification.md`; when they disagree, one of them is
+ * wrong and the run says so.
+ */
+function mergeMisencodedAlternatives(message, groups, from) {
+    for (let i = from; i + 1 < groups.length; i++) {
+        const left = groups[i];
+        const right = groups[i + 1];
+        if (left.alternative || right.alternative || left.members.length !== 1 || right.members.length !== 1) {
+            continue;
+        }
+        const a = message.partsById.get(left.members[0]);
+        const b = message.partsById.get(right.members[0]);
+        if (!a || !b) {
+            continue;
+        }
+        const plain = a.contentType === 'text/plain' ? a : b.contentType === 'text/plain' ? b : null;
+        const markup = a.contentType === 'text/html' ? a : b.contentType === 'text/html' ? b : null;
+        if (!plain || !markup) {
+            continue;
+        }
+        const left_ = (0, shingle_1.buildShingleIndex)((0, normalize_1.normaliseForMatch)(visibleTextOf(plain)));
+        const right_ = (0, shingle_1.buildShingleIndex)((0, normalize_1.normaliseForMatch)(visibleTextOf(markup)));
+        if (!(0, shingle_1.isMatchable)(left_) || !(0, shingle_1.isMatchable)(right_)) {
+            continue;
+        }
+        if ((0, shingle_1.containment)(left_, right_).containment < shingle_1.CONTAINMENT_THRESHOLD && (0, shingle_1.containment)(right_, left_).containment < shingle_1.CONTAINMENT_THRESHOLD) {
+            continue;
+        }
+        groups.splice(i, 2, { id: left.id, alternative: true, members: [...left.members, ...right.members] });
+        i--;
+    }
+}
+/**
+ * Required coverage groups for a parsed message.
+ *
+ * Written from the decision table rather than by calling `planBody()`. Sharing the implementation
+ * would make the oracle agree with the selection code by construction, which is exactly the
+ * property a gate must not have. The two are reviewed against the same table and are expected to
+ * agree; when they do not, one of them is wrong and the run says so.
+ */
+function requiredCoverageGroups(message) {
+    const groups = [];
+    const walk = (part, depth) => {
+        if (depth > 32 || part.disposition === 'attachment') {
+            return;
+        }
+        // Embedded messages are content for the caller to render, not the container's body.
+        if (part.rfc822 || part.embeddedRoot) {
+            return;
+        }
+        if (part.multipart === 'alternative') {
+            const members = [];
+            supportedLeavesUnder(message, part, members);
+            if (members.length) {
+                groups.push({ id: part.partId, alternative: true, members });
+            }
+            return;
+        }
+        const children = part.childIds.map(id => message.partsById.get(id)).filter(Boolean);
+        if (part.multipart === 'related') {
+            // Only the root carries body. Everything else is an inline resource.
+            const start = bareContentId(part.contentTypeParams.start);
+            const root = (start ? children.find(child => bareContentId(child.headers.find(entry => entry.key === 'content-id')?.value) === start) : undefined) ??
+                children[0];
+            if (root) {
+                walk(root, depth + 1);
+            }
+            return;
+        }
+        if (part.multipart) {
+            // mixed and everything unrecognised: ordered independent content, each body-bearing
+            // child is its own required group.
+            const before = groups.length;
+            for (const child of children) {
+                walk(child, depth + 1);
+            }
+            mergeMisencodedAlternatives(message, groups, before);
+            return;
+        }
+        if (isSupportedLeaf(part)) {
+            groups.push({ id: part.partId, alternative: false, members: [part.partId] });
+        }
+    };
+    const root = message.partsById.get(message.rootId) ?? message.parts[0];
+    if (root) {
+        walk(root, 0);
+    }
+    return groups;
+}
+/** Compare the required groups against the MIME parts the result actually names. */
+function coverageOf(groups, visibleParts, hiddenParts) {
+    const missing = [];
+    const alternativesNotEmitted = [];
+    const hiddenOnly = [];
+    let covered = 0;
+    for (const group of groups) {
+        const present = group.members.filter(member => visibleParts.has(member) || hiddenParts.has(member));
+        if (!present.length) {
+            missing.push(group.id);
+            continue;
+        }
+        covered++;
+        if (group.alternative) {
+            alternativesNotEmitted.push(...group.members.filter(member => !present.includes(member)));
+        }
+        if (present.every(member => !visibleParts.has(member))) {
+            hiddenOnly.push(group.id);
+        }
+    }
+    return { required: groups.length, covered, missing, alternativesNotEmitted, hiddenOnly };
+}
+//# sourceMappingURL=oracle.js.map

package/dist/eval/oracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"oracle.js","sourceRoot":"","sources":["../../src/eval/oracle.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;GAiBG;;AASH,0CAiBC;AAGD,8CASC;AAqBD,sDAOC;AAkID,wDAwDC;AAcD,gCAsBC;AA9RD,4CAAqD;AACrD,4CAAoD;AACpD,kDAAuD;AACvD,8CAAsG;AAGtG,sGAAsG;AACtG,SAAgB,eAAe,CAAC,OAAsB;IAClD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC;IAClF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,MAAM,GAAyB,IAAI,CAAC;QACxC,OAAO,MAAM,EAAE,QAAQ,EAAE,CAAC;YACtB,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACtE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxB,MAAM;YACV,CAAC;YACD,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,qGAAqG;AACrG,SAAgB,iBAAiB,CAAC,OAAsB;IACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,CACvB,IAAI,CAAC,EAAE,CACH,CAAC,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,CAAC;QACvE,IAAI,CAAC,WAAW,KAAK,YAAY;QACjC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QACrB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CACjC,CAAC;AACN,CAAC;AAWD;;;;;;;;;GASG;AACH,SAAgB,qBAAqB,CAAC,OAAsB;IACxD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,YAAY,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC/C,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,YAAY,GAAG,CAAC,EAAE,CAAC;AAC3F,CAAC;AAmBD;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,IAAc;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;IACrC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;QACnC,OAAO,IAAA,sBAAe,EAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IACD,OAAO,IAAA,uBAAgB,EAAC,GAAG,CAAC;SACvB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;SACjC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;SACzB,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,IAAc;IACnC,OAAO,CACH,CAAC,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,CAAC;QACvE,IAAI,CAAC,WAAW,KAAK,YAAY;QACjC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QACrB,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,CAC7B,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAsB,EAAE,IAAc,EAAE,IAAc,EAAE,KAAK,GAAG,CAAC;IAC3F,IAAI,KAAK,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtF,OAAO;IACX,CAAC;IACD,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO;IACX,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACR,oBAAoB,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAC,KAAgC;IACnD,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAClF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,2BAA2B,CAAC,OAAsB,EAAE,MAAuB,EAAE,IAAY;IAC9F,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAkB,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAkB,CAAC;QAC7C,IAAI,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnG,SAAS;QACb,CAAC;QACD,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAW,CAAC,CAAC;QAC3D,MAAM,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAW,CAAC,CAAC;QAC5D,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACX,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7F,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5F,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,2BAAiB,EAAC,IAAA,6BAAiB,EAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAA,2BAAiB,EAAC,IAAA,6BAAiB,EAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAA,qBAAW,EAAC,KAAK,CAAC,IAAI,CAAC,IAAA,qBAAW,EAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,SAAS;QACb,CAAC;QACD,IAAI,IAAA,qBAAW,EAAC,KAAK,EAAE,MAAM,CAAC,CAAC,WAAW,GAAG,+BAAqB,IAAI,IAAA,qBAAW,EAAC,MAAM,EAAE,KAAK,CAAC,CAAC,WAAW,GAAG,+BAAqB,EAAE,CAAC;YACnI,SAAS;QACb,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACtG,CAAC,EAAE,CAAC;IACR,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,sBAAsB,CAAC,OAAsB;IACzD,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,MAAM,IAAI,GAAG,CAAC,IAAc,EAAE,KAAa,EAAQ,EAAE;QACjD,IAAI,KAAK,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YAClD,OAAO;QACX,CAAC;QACD,oFAAoF;QACpF,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACnC,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,KAAK,aAAa,EAAE,CAAC;YACnC,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,oBAAoB,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC7C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAe,CAAC;QAElG,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC/B,qEAAqE;YACrE,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,IAAI,GACN,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,YAAY,CAAC,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACrI,QAAQ,CAAC,CAAC,CAAC,CAAC;YAChB,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1B,CAAC;YACD,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,oFAAoF;YACpF,mCAAmC;YACnC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC3B,CAAC;YACD,2BAA2B,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACrD,OAAO;QACX,CAAC;QAED,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,IAAI,EAAE,CAAC;QACP,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAaD,oFAAoF;AACpF,SAAgB,UAAU,CAAC,MAAgC,EAAE,YAAiC,EAAE,WAAgC;IAC5H,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,sBAAsB,GAAa,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACvB,SAAS;QACb,CAAC;QACD,OAAO,EAAE,CAAC;QACV,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,sBAAsB,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACrD,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,CAAC;AAC7F,CAAC"}

package/dist/index.d.ts

@@ -12,13 +12,16 @@ export * from './types';
 export { parseMessage } from './mime/parse';
 export type { ParsedMessage, MimePart } from './mime/model';
 export type { Block, BlockFeatures, BlockKind, Evidence } from './segment/types';
-export { DEFAULT_CONFIDENCE } from './classify/confidence';
+export { DEFAULT_CONFIDENCE, validateConfidenceConfig, type ConfidenceConfig } from './classify/confidence';
+export { validateParseOptions, DEFAULT_PARSE_OPTIONS } from './mime/model';
 export { buildHistoryDocument, matchBlocks, type HistoryDocument, type BlockMatch } from './match/history';
 export { corpusThreadResolver, folderThreadResolver, type ThreadResolver } from './corpus/thread-index';
-export { analyze, resolveHistory, senderTokensFrom, type MessageContent, type AnalyzeOptions, type AnalyzeResult } from './analyze';
+export { analyze, resolveHistory, senderTokensFrom, type BodySection, type MessageContent, type AnalyzeOptions, type AnalyzeResult } from './analyze';
+export { planBody, reportableExclusions, type BodyPlan, type ExclusionReason, type PlannedSection, type SectionRole } from './mime/select';
+export { loadPatterns, PatternAssetError } from './detect/patterns';
 export interface ExtractOptions {
     parse?: ParseOptions;
-    confidence?: ConfidenceConfig;
+    confidence?: Partial<ConfidenceConfig>;
     /**
      * Earlier messages in the thread, most recent first. Optional: extraction works without any
      * history, and history only ever adds evidence.

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAiB,KAAK,YAAY,EAAsB,MAAM,cAAc,CAAC;AACpF,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,MAAM,EAAgB,MAAM,SAAS,CAAC;AAE9F,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAEpI,MAAM,WAAW,cAAc;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;OAGG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAC9C,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACrD,qCAAqC;IACrC,WAAW,CAAC,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,MAAM,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,EAAE,MAAM,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAC;gBAAC,QAAQ,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;YAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC;SACnB,EAAE,CAAC;KACP,CAAC;CACL;AAkFD,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,CA+C1G"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,YAAY,EAA4C,MAAM,cAAc,CAAC;AAG3F,OAAO,EAA4B,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,MAAM,EAAgB,MAAM,SAAS,CAAC;AAE9F,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC5G,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AACtJ,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAC3I,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAWpE,MAAM,WAAW,cAAc;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC;;;OAGG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAC9C,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACrD,qCAAqC;IACrC,WAAW,CAAC,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,MAAM,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,EAAE,MAAM,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAC;gBAAC,QAAQ,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;YAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC;SACnB,EAAE,CAAC;KACP,CAAC;CACL;AA2CD,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,CA2G1G"}

package/dist/index.js

@@ -20,16 +20,24 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.senderTokensFrom = exports.resolveHistory = exports.analyze = exports.folderThreadResolver = exports.corpusThreadResolver = exports.matchBlocks = exports.buildHistoryDocument = exports.DEFAULT_CONFIDENCE = exports.parseMessage = void 0;
+exports.PatternAssetError = exports.loadPatterns = exports.reportableExclusions = exports.planBody = exports.senderTokensFrom = exports.resolveHistory = exports.analyze = exports.folderThreadResolver = exports.corpusThreadResolver = exports.matchBlocks = exports.buildHistoryDocument = exports.DEFAULT_PARSE_OPTIONS = exports.validateParseOptions = exports.validateConfidenceConfig = exports.DEFAULT_CONFIDENCE = exports.parseMessage = void 0;
 exports.extract = extract;
 const parse_1 = require("./mime/parse");
+const model_1 = require("./mime/model");
+const select_1 = require("./mime/select");
+const patterns_1 = require("./detect/patterns");
+const confidence_1 = require("./classify/confidence");
 const analyze_1 = require("./analyze");
 const types_1 = require("./types");
 __exportStar(require("./types"), exports);
 var parse_2 = require("./mime/parse");
 Object.defineProperty(exports, "parseMessage", { enumerable: true, get: function () { return parse_2.parseMessage; } });
-var confidence_1 = require("./classify/confidence");
-Object.defineProperty(exports, "DEFAULT_CONFIDENCE", { enumerable: true, get: function () { return confidence_1.DEFAULT_CONFIDENCE; } });
+var confidence_2 = require("./classify/confidence");
+Object.defineProperty(exports, "DEFAULT_CONFIDENCE", { enumerable: true, get: function () { return confidence_2.DEFAULT_CONFIDENCE; } });
+Object.defineProperty(exports, "validateConfidenceConfig", { enumerable: true, get: function () { return confidence_2.validateConfidenceConfig; } });
+var model_2 = require("./mime/model");
+Object.defineProperty(exports, "validateParseOptions", { enumerable: true, get: function () { return model_2.validateParseOptions; } });
+Object.defineProperty(exports, "DEFAULT_PARSE_OPTIONS", { enumerable: true, get: function () { return model_2.DEFAULT_PARSE_OPTIONS; } });
 var history_1 = require("./match/history");
 Object.defineProperty(exports, "buildHistoryDocument", { enumerable: true, get: function () { return history_1.buildHistoryDocument; } });
 Object.defineProperty(exports, "matchBlocks", { enumerable: true, get: function () { return history_1.matchBlocks; } });
@@ -40,50 +48,20 @@ var analyze_2 = require("./analyze");
 Object.defineProperty(exports, "analyze", { enumerable: true, get: function () { return analyze_2.analyze; } });
 Object.defineProperty(exports, "resolveHistory", { enumerable: true, get: function () { return analyze_2.resolveHistory; } });
 Object.defineProperty(exports, "senderTokensFrom", { enumerable: true, get: function () { return analyze_2.senderTokensFrom; } });
+var select_2 = require("./mime/select");
+Object.defineProperty(exports, "planBody", { enumerable: true, get: function () { return select_2.planBody; } });
+Object.defineProperty(exports, "reportableExclusions", { enumerable: true, get: function () { return select_2.reportableExclusions; } });
+var patterns_2 = require("./detect/patterns");
+Object.defineProperty(exports, "loadPatterns", { enumerable: true, get: function () { return patterns_2.loadPatterns; } });
+Object.defineProperty(exports, "PatternAssetError", { enumerable: true, get: function () { return patterns_2.PatternAssetError; } });
 /**
- * Choose the text and HTML parts that hold the message body.
+ * Classifier assets are validated when the package is required, not when a message arrives.
  *
- * The largest candidate of each type wins. A message can carry several text parts, for example a
- * gateway appended disclaimer in `multipart/mixed(alternative(P,H), P)`, and the body is the large
- * one. Attachments and parts inside a forwarded `message/rfc822` are excluded: an embedded message
- * is content to be classified, not the container's own body.
+ * Missing or corrupt pattern files are an installation defect. Discovering that on the thousandth
+ * message, as one more degraded result, is far worse than discovering it at startup: every message
+ * until then came back looking like new content, because nothing could be recognised as history.
  */
-function selectBodyParts(message) {
-    const insideEmbedded = new Set();
-    for (const part of message.parts) {
-        if (!part.rfc822 && !part.embeddedRoot) {
-            continue;
-        }
-        for (const other of message.parts) {
-            let cursor = other;
-            while (cursor?.parentId) {
-                if (cursor.parentId === part.partId) {
-                    insideEmbedded.add(other.partId);
-                    break;
-                }
-                cursor = message.partsById.get(cursor.parentId);
-            }
-        }
-    }
-    let text = null;
-    let html = null;
-    for (const part of message.parts) {
-        if (part.disposition === 'attachment' || !part.decoded || insideEmbedded.has(part.partId)) {
-            continue;
-        }
-        if (part.contentType === 'text/plain') {
-            if (!text || part.decoded.text.length > (text.decoded?.text.length ?? 0)) {
-                text = part;
-            }
-        }
-        else if (part.contentType === 'text/html') {
-            if (!html || part.decoded.text.length > (html.decoded?.text.length ?? 0)) {
-                html = part;
-            }
-        }
-    }
-    return { text, html };
-}
+(0, patterns_1.loadPatterns)();
 /**
  * Extract the conversationally relevant content of a raw RFC 5322 message.
  *
@@ -105,7 +83,9 @@ function degraded(text, warnings) {
                     confidence: 0.5,
                     visibility: 'show',
                     reasons: ['degraded_to_unknown'],
-                    source: { mimePart: 'TEXT', representation: 'text', blockIndex: 0, start: 0, end: text.length }
+                    // No MIME tree was produced, so there is no part to name. Inventing one, which
+                    // this used to do, points a reviewer at a part that does not exist.
+                    source: { kind: 'raw-message', start: 0, end: text.length }
                 }
             ]
             : [],
@@ -115,39 +95,93 @@ function degraded(text, warnings) {
         originalAvailable: true,
         primaryRepresentation: 'none',
         extractorVersion: types_1.EXTRACTOR_VERSION,
-        historyUsed: false
+        historyUsed: false,
+        // Threading headers are exactly what a failed parse could not read. Reply status is left
+        // false and the uncertainty is made observable, rather than guessed at with a substring
+        // search over the raw bytes.
+        isReply: false
     };
 }
 async function extract(raw, options = {}) {
+    // Assets first, then options. Both are checked outside the fail-soft path, because neither is
+    // a property of the message and both would otherwise repeat silently on every one of them.
+    (0, patterns_1.loadPatterns)();
+    // Validated before any work, so a bad option fails on the first message rather than producing
+    // subtly wrong results on every message.
+    const parseOptions = (0, model_1.validateParseOptions)(options.parse);
+    const confidence = (0, confidence_1.validateConfidenceConfig)(options.confidence);
     const buffer = typeof raw === 'string' ? Buffer.from(raw) : raw;
+    if (buffer.length > parseOptions.maxMessageBytes) {
+        // A circuit breaker the caller asked for. Not an error: an oversized message is input, and
+        // input degrades rather than throwing. Nothing was read, so nothing is returned, and the
+        // caller still holds the original.
+        return degraded('', [
+            { code: 'input_limit_exceeded', detail: `maxMessageBytes (${parseOptions.maxMessageBytes}), message was ${buffer.length}` },
+            { code: 'extraction_degraded' },
+            { code: 'reply_status_unknown' }
+        ]);
+    }
     let message;
     try {
-        message = await (0, parse_1.parseMessage)(buffer, options.parse);
+        message = await (0, parse_1.parseMessage)(buffer, parseOptions);
     }
     catch (err) {
         return degraded(buffer.toString('utf8'), [
             { code: 'mime_parse_error', detail: err instanceof Error ? err.message.slice(0, 200) : 'unknown' },
-            { code: 'extraction_degraded' }
+            { code: 'extraction_degraded' },
+            { code: 'reply_status_unknown' }
         ]);
     }
-    const parts = selectBodyParts(message);
+    const plan = (0, select_1.planBody)(message);
+    // A tree that failed to parse must not come back as nothing.
+    //
+    // `parseMessage()` never rejects: it records the failure as a warning and returns whatever it
+    // salvaged. When it salvaged no parts at all, the body plan is empty, and the result used to be
+    // zero segments plus `no_body_parts` even though the raw message plainly held text. That is the
+    // same defect as the malformed HTML fallback, one stage earlier, and with the same shape: total
+    // content loss whose only trace is a warning.
+    if (!plan.sections.length && message.warnings.some(warning => warning.code === 'mime_parse_error')) {
+        const text = buffer.toString('utf8');
+        if (text.trim()) {
+            return degraded(text, [
+                ...message.warnings,
+                { code: 'extraction_degraded', detail: 'no MIME body could be recovered' },
+                { code: 'reply_status_unknown' }
+            ]);
+        }
+    }
+    // Every eligible inline body part the plan leaves out is named. Silent substitution is the
+    // failure mode this replaced: a long inline footer used to displace a short authored reply with
+    // nothing in the result to say so. Unused alternative members, attachments and embedded
+    // messages are excluded by a written rule and are not reported, because they are not losses.
+    const omissions = (0, select_1.reportableExclusions)(plan).map(entry => ({
+        code: 'body_section_not_selected',
+        mimePart: entry.partId,
+        detail: entry.reason
+    }));
+    // The whole plan goes through, in document order. Composition happens inside analyze() so that
+    // a caller who already has a parsed multipart message can express exactly the same thing.
+    const sections = plan.sections.map(section => ({
+        text: section.text?.decoded?.text ?? null,
+        html: section.html?.decoded?.text ?? null,
+        ...(section.text ? { textPartId: section.text.partId } : {}),
+        ...(section.html ? { htmlPartId: section.html.partId } : {}),
+        role: section.role
+    }));
     const result = await (0, analyze_1.analyze)({
-        text: parts.text?.decoded?.text ?? null,
-        html: parts.html?.decoded?.text ?? null,
+        sections,
         subject: message.headers.subject,
         from: message.headers.from,
         inReplyTo: message.headers.inReplyTo,
-        references: message.headers.references,
-        ...(parts.text ? { textPartId: parts.text.partId } : {}),
-        ...(parts.html ? { htmlPartId: parts.html.partId } : {})
+        references: message.headers.references
     }, {
-        ...(options.confidence ? { confidence: options.confidence } : {}),
+        confidence,
         ...(options.history ? { history: options.history } : {}),
         ...(options.historyDocuments ? { historyDocuments: options.historyDocuments } : {}),
         ...(options.explain ? { explain: true } : {})
     });
     // MIME level warnings belong to the message, not to the body analysis.
-    const warnings = [...message.warnings, ...result.warnings];
+    const warnings = [...message.warnings, ...omissions, ...result.warnings];
     return {
         segments: result.segments,
         hiddenSegments: result.hiddenSegments,
@@ -157,6 +191,10 @@ async function extract(raw, options = {}) {
         primaryRepresentation: result.primaryRepresentation,
         extractorVersion: result.extractorVersion,
         historyUsed: result.historyUsed,
+        isReply: result.isReply,
+        // `satisfies` rather than a bare return: an omitted mandatory field then fails the build
+        // instead of quietly disappearing from the public result, which is how `isReply` went
+        // missing from this entry point while the README documented it.
         ...(result.diagnostics ? { diagnostics: { ...result.diagnostics, structure: message.structure } } : {})
     };
 }

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;AAkIH,0BA+CC;AA/KD,wCAA4C;AAI5C,uCAAoC;AACpC,mCAA8F;AAE9F,0CAAwB;AACxB,sCAA4C;AAAnC,qGAAA,YAAY,OAAA;AAGrB,oDAA2D;AAAlD,gHAAA,kBAAkB,OAAA;AAC3B,2CAA2G;AAAlG,+GAAA,oBAAoB,OAAA;AAAE,sGAAA,WAAW,OAAA;AAC1C,sDAAwG;AAA/F,oHAAA,oBAAoB,OAAA;AAAE,oHAAA,oBAAoB,OAAA;AACnD,qCAAoI;AAA3H,kGAAA,OAAO,OAAA;AAAE,yGAAA,cAAc,OAAA;AAAE,2GAAA,gBAAgB,OAAA;AAkClD;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,OAAsB;IAC3C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrC,SAAS;QACb,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,MAAM,GAAyB,KAAK,CAAC;YACzC,OAAO,MAAM,EAAE,QAAQ,EAAE,CAAC;gBACtB,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;oBAClC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjC,MAAM;gBACV,CAAC;gBACD,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACxF,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;gBACvE,IAAI,GAAG,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;gBACvE,IAAI,GAAG,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,QAAmB;IAC/C,OAAO;QACH,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE;YACjB,CAAC,CAAC;gBACI;oBACI,IAAI,EAAE,SAAS;oBACf,IAAI;oBACJ,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,MAAM;oBAClB,OAAO,EAAE,CAAC,qBAAqB,CAAC;oBAChC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE;iBAClG;aACJ;YACH,CAAC,CAAC,EAAE;QACR,cAAc,EAAE,EAAE;QAClB,iBAAiB,EAAE,GAAG;QACtB,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM;QAC7B,gBAAgB,EAAE,yBAAiB;QACnC,WAAW,EAAE,KAAK;KACrB,CAAC;AACN,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,GAAoB,EAAE,UAA0B,EAAE;IAC5E,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEhE,IAAI,OAAsB,CAAC;IAC3B,IAAI,CAAC;QACD,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACrC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE;YAClG,EAAE,IAAI,EAAE,qBAAqB,EAAE;SAClC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAO,EACxB;QACI,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACvC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACvC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO;QAChC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;QAC1B,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS;QACpC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;QACtC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,EACD;QACI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChD,CACJ,CAAC;IAEF,uEAAuE;IACvE,MAAM,QAAQ,GAAc,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEtE,OAAO;QACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;QACnD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1G,CAAC;AACN,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;AAyGH,0BA2GC;AAlND,wCAA4C;AAC5C,wCAA2F;AAC3F,0CAA+D;AAC/D,gDAAiD;AACjD,sDAAwF;AAExF,uCAAsD;AACtD,mCAA8F;AAE9F,0CAAwB;AACxB,sCAA4C;AAAnC,qGAAA,YAAY,OAAA;AAGrB,oDAA4G;AAAnG,gHAAA,kBAAkB,OAAA;AAAE,sHAAA,wBAAwB,OAAA;AACrD,sCAA2E;AAAlE,6GAAA,oBAAoB,OAAA;AAAE,8GAAA,qBAAqB,OAAA;AACpD,2CAA2G;AAAlG,+GAAA,oBAAoB,OAAA;AAAE,sGAAA,WAAW,OAAA;AAC1C,sDAAwG;AAA/F,oHAAA,oBAAoB,OAAA;AAAE,oHAAA,oBAAoB,OAAA;AACnD,qCAAsJ;AAA7I,kGAAA,OAAO,OAAA;AAAE,yGAAA,cAAc,OAAA;AAAE,2GAAA,gBAAgB,OAAA;AAClD,wCAA2I;AAAlI,kGAAA,QAAQ,OAAA;AAAE,8GAAA,oBAAoB,OAAA;AACvC,8CAAoE;AAA3D,wGAAA,YAAY,OAAA;AAAE,6GAAA,iBAAiB,OAAA;AAExC;;;;;;GAMG;AACH,IAAA,uBAAY,GAAE,CAAC;AAkCf;;;;;;;;;;GAUG;AACH,SAAS,QAAQ,CAAC,IAAY,EAAE,QAAmB;IAC/C,OAAO;QACH,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE;YACjB,CAAC,CAAC;gBACI;oBACI,IAAI,EAAE,SAAS;oBACf,IAAI;oBACJ,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,MAAM;oBAClB,OAAO,EAAE,CAAC,qBAAqB,CAAC;oBAChC,+EAA+E;oBAC/E,oEAAoE;oBACpE,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE;iBAC9D;aACJ;YACH,CAAC,CAAC,EAAE;QACR,cAAc,EAAE,EAAE;QAClB,iBAAiB,EAAE,GAAG;QACtB,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM;QAC7B,gBAAgB,EAAE,yBAAiB;QACnC,WAAW,EAAE,KAAK;QAClB,yFAAyF;QACzF,wFAAwF;QACxF,6BAA6B;QAC7B,OAAO,EAAE,KAAK;KACU,CAAC;AACjC,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,GAAoB,EAAE,UAA0B,EAAE;IAC5E,8FAA8F;IAC9F,2FAA2F;IAC3F,IAAA,uBAAY,GAAE,CAAC;IAEf,8FAA8F;IAC9F,yCAAyC;IACzC,MAAM,YAAY,GAAG,IAAA,4BAAoB,EAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,IAAA,qCAAwB,EAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEhE,IAAI,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC,eAAe,EAAE,CAAC;QAC/C,2FAA2F;QAC3F,yFAAyF;QACzF,mCAAmC;QACnC,OAAO,QAAQ,CAAC,EAAE,EAAE;YAChB,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,oBAAoB,YAAY,CAAC,eAAe,kBAAkB,MAAM,CAAC,MAAM,EAAE,EAAE;YAC3H,EAAE,IAAI,EAAE,qBAAqB,EAAE;YAC/B,EAAE,IAAI,EAAE,sBAAsB,EAAE;SACnC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,OAAsB,CAAC;IAC3B,IAAI,CAAC;QACD,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACrC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE;YAClG,EAAE,IAAI,EAAE,qBAAqB,EAAE;YAC/B,EAAE,IAAI,EAAE,sBAAsB,EAAE;SACnC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,IAAI,GAAG,IAAA,iBAAQ,EAAC,OAAO,CAAC,CAAC;IAE/B,6DAA6D;IAC7D,EAAE;IACF,8FAA8F;IAC9F,gGAAgG;IAChG,gGAAgG;IAChG,gGAAgG;IAChG,8CAA8C;IAC9C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,kBAAkB,CAAC,EAAE,CAAC;QACjG,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC,IAAI,EAAE;gBAClB,GAAG,OAAO,CAAC,QAAQ;gBACnB,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,iCAAiC,EAAE;gBAC1E,EAAE,IAAI,EAAE,sBAAsB,EAAE;aACnC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,2FAA2F;IAC3F,gGAAgG;IAChG,wFAAwF;IACxF,6FAA6F;IAC7F,MAAM,SAAS,GAAc,IAAA,6BAAoB,EAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClE,IAAI,EAAE,2BAAoC;QAC1C,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;KACvB,CAAC,CAAC,CAAC;IAEJ,+FAA+F;IAC/F,0FAA0F;IAC1F,MAAM,QAAQ,GAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1D,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACzC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI;QACzC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,IAAI,EAAE,OAAO,CAAC,IAAI;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAO,EACxB;QACI,QAAQ;QACR,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO;QAChC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;QAC1B,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS;QACpC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;KACzC,EACD;QACI,UAAU;QACV,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChD,CACJ,CAAC;IAEF,uEAAuE;IACvE,MAAM,QAAQ,GAAc,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEpF,OAAO;QACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,QAAQ;QACR,iBAAiB,EAAE,IAAI;QACvB,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;QACnD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,yFAAyF;QACzF,sFAAsF;QACtF,gEAAgE;QAChE,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChF,CAAC;AAChC,CAAC"}

package/dist/match/history.d.ts

@@ -59,4 +59,16 @@ export declare function buildHistoryDocumentFromContent(text: string, html: stri
 export declare function matchBlocks(blocks: readonly Block[], history: readonly HistoryDocument[]): BlockMatch[];
 /** Convenience view for the classifier, which only needs to know which indices matched. */
 export declare function matchedIndices(matches: readonly BlockMatch[]): Set<number>;
+/**
+ * Containment at or above which a match needs no corroboration from markup.
+ *
+ * The ordinary threshold admits a paraphrase or a heavily edited quote, which is right when markup
+ * already says the region is history and the match only confirms it. Identifying history that
+ * carries no markup at all is a stronger claim and takes a stronger match. Measured across the
+ * reference set the ratio of correct to incorrect reclassification is best here, at almost twelve
+ * to one.
+ */
+export declare const STRONG_CONTAINMENT = 0.95;
+/** Matches strong enough to identify history that carries no markup of its own. */
+export declare function strongMatchedIndices(matches: readonly BlockMatch[]): Set<number>;
 //# sourceMappingURL=history.d.ts.map

package/dist/match/history.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAsE,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAGlH,MAAM,WAAW,eAAe;IAC5B,kEAAkE;IAClE,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAkBxH;AAED;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,eAAe,CAU1H;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,eAAe,EAAE,GAAG,UAAU,EAAE,CA4CvG;AASD,2FAA2F;AAC3F,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAE1E"}
+{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAsE,KAAK,YAAY,EAAE,MAAM,WAAW,CAAC;AAGlH,MAAM,WAAW,eAAe;IAC5B,kEAAkE;IAClE,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAkBxH;AAED;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,eAAe,CAU1H;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,eAAe,EAAE,GAAG,UAAU,EAAE,CA4CvG;AASD,2FAA2F;AAC3F,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAE1E;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAEvC,mFAAmF;AACnF,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAEhF"}

package/dist/match/history.js

@@ -17,10 +17,12 @@
  * threshold is required. A non match never demotes anything.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.STRONG_CONTAINMENT = void 0;
 exports.buildHistoryDocument = buildHistoryDocument;
 exports.buildHistoryDocumentFromContent = buildHistoryDocumentFromContent;
 exports.matchBlocks = matchBlocks;
 exports.matchedIndices = matchedIndices;
+exports.strongMatchedIndices = strongMatchedIndices;
 const parse_1 = require("../mime/parse");
 const text_1 = require("../canonical/text");
 const html_1 = require("../canonical/html");
@@ -125,4 +127,18 @@ function betterMatch(candidate, current) {
 function matchedIndices(matches) {
     return new Set(matches.map(match => match.blockIndex));
 }
+/**
+ * Containment at or above which a match needs no corroboration from markup.
+ *
+ * The ordinary threshold admits a paraphrase or a heavily edited quote, which is right when markup
+ * already says the region is history and the match only confirms it. Identifying history that
+ * carries no markup at all is a stronger claim and takes a stronger match. Measured across the
+ * reference set the ratio of correct to incorrect reclassification is best here, at almost twelve
+ * to one.
+ */
+exports.STRONG_CONTAINMENT = 0.95;
+/** Matches strong enough to identify history that carries no markup of its own. */
+function strongMatchedIndices(matches) {
+    return new Set(matches.filter(match => match.exact || match.containment >= exports.STRONG_CONTAINMENT).map(match => match.blockIndex));
+}
 //# sourceMappingURL=history.js.map

package/dist/match/history.js.map

@@ -1 +1 @@
-{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;AAmCH,oDAkBC;AASD,0EAUC;AAQD,kCA4CC;AAUD,wCAEC;AAtID,yCAA6C;AAC7C,4CAAqD;AACrD,4CAAoD;AAEpD,uCAAkH;AAClH,2CAAgD;AAqBhD;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,GAAoB,EAAE,EAAU,EAAE,QAAiB;IAC1F,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACrD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,+BAA+B,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU,EAAE,QAAiB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,MAAwB,EAAE,OAAmC;IACrF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,2BAAiB,EAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAA,qBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;YACtB,wFAAwF;YACxF,SAAS;QACb,CAAC;QAED,IAAI,IAAI,GAAsB,IAAI,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC7B,yEAAyE;YACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,qBAAW,EAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC;YACzE,IAAI,KAAK,GAAG,+BAAqB,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAe;gBAC1B,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,KAAK;gBAClB,KAAK;aACR,CAAC;YACF,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBACxC,IAAI,GAAG,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,SAAqB,EAAE,OAAmB;IAC3D,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,SAAS,CAAC,QAAQ,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;AACvD,CAAC;AAED,2FAA2F;AAC3F,SAAgB,cAAc,CAAC,OAA8B;IACzD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AAC3D,CAAC"}
+{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;;AAmCH,oDAkBC;AASD,0EAUC;AAQD,kCA4CC;AAUD,wCAEC;AAcD,oDAEC;AAtJD,yCAA6C;AAC7C,4CAAqD;AACrD,4CAAoD;AAEpD,uCAAkH;AAClH,2CAAgD;AAqBhD;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,GAAoB,EAAE,EAAU,EAAE,QAAiB;IAC1F,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACrD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,+BAA+B,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU,EAAE,QAAiB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,MAAwB,EAAE,OAAmC;IACrF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,2BAAiB,EAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAA,qBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;YACtB,wFAAwF;YACxF,SAAS;QACb,CAAC;QAED,IAAI,IAAI,GAAsB,IAAI,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC7B,yEAAyE;YACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,qBAAW,EAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC;YACzE,IAAI,KAAK,GAAG,+BAAqB,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAe;gBAC1B,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,KAAK;gBAClB,KAAK;aACR,CAAC;YACF,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBACxC,IAAI,GAAG,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,SAAqB,EAAE,OAAmB;IAC3D,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,SAAS,CAAC,QAAQ,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;AACvD,CAAC;AAED,2FAA2F;AAC3F,SAAgB,cAAc,CAAC,OAA8B;IACzD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;;;GAQG;AACU,QAAA,kBAAkB,GAAG,IAAI,CAAC;AAEvC,mFAAmF;AACnF,SAAgB,oBAAoB,CAAC,OAA8B;IAC/D,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,0BAAkB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AACnI,CAAC"}