@postalsys/email-content 0.0.1

package/dist/segment/text-blocks.js

@@ -0,0 +1,204 @@
+"use strict";
+/**
+ * Stage 3: plain text block segmentation.
+ *
+ * Blocks tile the canonical text completely and without overlap. Every character belongs to exactly
+ * one block, including blank lines and trailing whitespace, so that concatenating block texts
+ * reproduces the input exactly. That is maintained by construction here, and asserted as a property
+ * test, because it is what guarantees content cannot vanish between stages.
+ *
+ * Boundaries are placed at:
+ *   - a change of quote depth
+ *   - a blank line, which separates paragraphs
+ *   - a reply attribution line, which becomes its own block
+ *   - a forward marker, which becomes its own block
+ *   - a signature delimiter, which becomes its own block
+ *   - a rule separator
+ *   - the start of an embedded header block
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.segmentTextBlocks = segmentTextBlocks;
+const text_1 = require("../canonical/text");
+const disclaimer_1 = require("../detect/disclaimer");
+const forward_header_1 = require("../detect/forward-header");
+const reply_header_1 = require("../detect/reply-header");
+const signature_1 = require("../detect/signature");
+const features_1 = require("./features");
+/** Strip quote prefixes and trim surrounding blank lines for display and matching. */
+function contentOf(lines) {
+    let start = 0;
+    let end = lines.length;
+    while (start < end && lines[start].blank) {
+        start++;
+    }
+    while (end > start && lines[end - 1].blank) {
+        end--;
+    }
+    return lines
+        .slice(start, end)
+        .map(line => line.content)
+        .join('\n');
+}
+function classifyKind(lines, forced) {
+    if (forced) {
+        return forced;
+    }
+    const body = lines.filter(line => !line.blank);
+    if (!body.length) {
+        return 'blank';
+    }
+    if (body[0].depth > 0) {
+        return 'quote';
+    }
+    const contents = body.map(line => line.content.trim());
+    if (contents.every(line => /^([-*+•‣◦]|\d+[.)])\s+/.test(line))) {
+        return 'list';
+    }
+    // A run of lines with aligned pipe or multi-space columns reads as a table in plain text.
+    if (contents.length > 1 && contents.every(line => line.includes('|') || /\S {3,}\S/.test(line))) {
+        return 'table';
+    }
+    return 'paragraph';
+}
+/**
+ * Segment a plain text part into blocks.
+ *
+ * @param raw decoded text of the part
+ * @param mimePart part id for traceability
+ */
+function segmentTextBlocks(raw, mimePart) {
+    const canonical = (0, text_1.canonicaliseText)(raw);
+    const lines = canonical.lines;
+    if (!lines.length) {
+        return { canonical, blocks: [] };
+    }
+    const contents = lines.map(line => line.content);
+    const boundaries = new Map();
+    const addBoundary = (boundary) => {
+        const existing = boundaries.get(boundary.line);
+        if (!existing || (boundary.evidence?.strength ?? 0) > (existing.evidence?.strength ?? 0)) {
+            boundaries.set(boundary.line, boundary);
+        }
+    };
+    // Line level anchors.
+    for (const line of lines) {
+        if (line.blank) {
+            continue;
+        }
+        const attribution = (0, reply_header_1.detectAttributionLine)(line.content);
+        if (attribution) {
+            addBoundary({ line: line.index, kind: 'reply_header', evidence: attribution, single: true });
+            continue;
+        }
+        const forward = (0, forward_header_1.detectForwardMarker)(line.content);
+        if (forward) {
+            addBoundary({ line: line.index, kind: 'forward_header', evidence: forward, single: true });
+            continue;
+        }
+        const signature = (0, signature_1.detectSignatureDelimiter)(line.content);
+        if (signature) {
+            addBoundary({ line: line.index, kind: 'separator', evidence: signature, single: true });
+            continue;
+        }
+        const rule = (0, reply_header_1.detectRuleSeparator)(line.content);
+        if (rule) {
+            addBoundary({ line: line.index, kind: 'separator', evidence: rule, single: true });
+        }
+    }
+    // Embedded header blocks, scanning repeatedly so that several can be found in one body.
+    let cursor = 0;
+    while (cursor < lines.length) {
+        const match = (0, reply_header_1.detectQuotedHeaderBlock)(contents, cursor);
+        if (!match) {
+            break;
+        }
+        addBoundary({ line: match.startLine, kind: 'quoted_header_block', evidence: match.evidence });
+        // The block ends after the last field line, so the next block starts there.
+        if (match.endLine + 1 < lines.length) {
+            const next = boundaries.get(match.endLine + 1);
+            if (!next) {
+                addBoundary({ line: match.endLine + 1 });
+            }
+        }
+        cursor = match.endLine + 1;
+    }
+    // Structural boundaries: depth changes and blank line separations.
+    let previousDepth = lines[0].depth;
+    let sawBlank = false;
+    for (const line of lines) {
+        if (line.index === 0) {
+            continue;
+        }
+        if (line.blank) {
+            sawBlank = true;
+            continue;
+        }
+        const forcedSingleBefore = boundaries.get(line.index - 1)?.single;
+        if (line.depth !== previousDepth || sawBlank || forcedSingleBefore) {
+            if (!boundaries.has(line.index)) {
+                addBoundary({ line: line.index });
+            }
+        }
+        previousDepth = line.depth;
+        sawBlank = false;
+    }
+    // A single line boundary must also close, so the following line starts a new block.
+    for (const boundary of [...boundaries.values()]) {
+        if (boundary.single && boundary.line + 1 < lines.length && !boundaries.has(boundary.line + 1)) {
+            boundaries.set(boundary.line + 1, { line: boundary.line + 1 });
+        }
+    }
+    const starts = [...new Set([0, ...boundaries.keys()])].sort((a, b) => a - b);
+    const blocks = [];
+    for (let i = 0; i < starts.length; i++) {
+        const startLine = starts[i];
+        const endLine = i + 1 < starts.length ? starts[i + 1] : lines.length;
+        const slice = lines.slice(startLine, endLine);
+        if (!slice.length) {
+            continue;
+        }
+        const first = slice[0];
+        const last = slice[slice.length - 1];
+        // Include the newline that terminates the last line, unless this is the end of the text.
+        const end = endLine < lines.length ? lines[endLine].start : last.end;
+        const boundary = boundaries.get(startLine);
+        blocks.push({
+            index: blocks.length,
+            kind: classifyKind(slice, boundary?.kind),
+            representation: 'text',
+            mimePart,
+            text: canonical.text.slice(first.start, end),
+            content: contentOf(slice),
+            start: first.start,
+            end,
+            features: (0, features_1.computeFeatures)(contentOf(slice), {
+                index: blocks.length,
+                total: starts.length,
+                start: first.start,
+                totalChars: canonical.text.length,
+                quoteDepth: Math.max(...slice.map(line => line.depth))
+            }),
+            evidence: boundary?.evidence ? [boundary.evidence] : []
+        });
+    }
+    // Block level evidence, once positions are known.
+    for (const block of blocks) {
+        block.features.indexFromEnd = blocks.length - 1 - block.index;
+        if (block.kind === 'blank') {
+            continue;
+        }
+        const blockLines = block.content.split('\n');
+        for (const evidence of (0, signature_1.detectSignaturePhrases)(blockLines)) {
+            block.evidence.push(evidence);
+        }
+        const shape = (0, signature_1.detectSignatureShape)(block.features);
+        if (shape) {
+            block.evidence.push(shape);
+        }
+        for (const evidence of (0, disclaimer_1.detectDisclaimer)(block.content, block.features)) {
+            block.evidence.push(evidence);
+        }
+    }
+    return { canonical, blocks };
+}
+//# sourceMappingURL=text-blocks.js.map

package/dist/segment/text-blocks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-blocks.js","sourceRoot":"","sources":["../../src/segment/text-blocks.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;AAgEH,8CAiJC;AA/MD,4CAAwF;AACxF,qDAAwD;AACxD,6DAA+D;AAC/D,yDAA6G;AAC7G,mDAA6G;AAC7G,yCAA6C;AAa7C,sFAAsF;AACtF,SAAS,SAAS,CAAC,KAA0B;IACzC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,OAAO,KAAK,GAAG,GAAG,IAAK,KAAK,CAAC,KAAK,CAAc,CAAC,KAAK,EAAE,CAAC;QACrD,KAAK,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,IAAK,KAAK,CAAC,GAAG,GAAG,CAAC,CAAc,CAAC,KAAK,EAAE,CAAC;QACvD,GAAG,EAAE,CAAC;IACV,CAAC;IACD,OAAO,KAAK;SACP,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;SACjB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;SACzB,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,SAAS,YAAY,CAAC,KAA0B,EAAE,MAA6B;IAC3E,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,IAAK,IAAI,CAAC,CAAC,CAAc,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,0FAA0F;IAC1F,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,SAAS,GAAG,IAAA,uBAAgB,EAAC,GAAG,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAE9B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC/C,MAAM,WAAW,GAAG,CAAC,QAAkB,EAAE,EAAE;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;YACvF,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC,CAAC;IAEF,sBAAsB;IACtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,SAAS;QACb,CAAC;QACD,MAAM,WAAW,GAAG,IAAA,oCAAqB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,WAAW,EAAE,CAAC;YACd,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7F,SAAS;QACb,CAAC;QACD,MAAM,OAAO,GAAG,IAAA,oCAAmB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,OAAO,EAAE,CAAC;YACV,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3F,SAAS;QACb,CAAC;QACD,MAAM,SAAS,GAAG,IAAA,oCAAwB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,SAAS,EAAE,CAAC;YACZ,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACxF,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,kCAAmB,EAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,IAAI,EAAE,CAAC;YACP,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,wFAAwF;IACxF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAA,sCAAuB,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM;QACV,CAAC;QACD,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9F,4EAA4E;QAC5E,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;QACL,CAAC;QACD,MAAM,GAAG,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,mEAAmE;IACnE,IAAI,aAAa,GAAI,KAAK,CAAC,CAAC,CAAc,CAAC,KAAK,CAAC;IACjD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACnB,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,QAAQ,GAAG,IAAI,CAAC;YAChB,SAAS;QACb,CAAC;QACD,MAAM,kBAAkB,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC;QAClE,IAAI,IAAI,CAAC,KAAK,KAAK,aAAa,IAAI,QAAQ,IAAI,kBAAkB,EAAE,CAAC;YACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QACD,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,QAAQ,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,oFAAoF;IACpF,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;QAC9C,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5F,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7E,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;QACtC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAY,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACjF,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAa,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAa,CAAC;QACjD,yFAAyF;QACzF,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAE,KAAK,CAAC,OAAO,CAAc,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACnF,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE3C,MAAM,CAAC,IAAI,CAAC;YACR,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC;YACzC,cAAc,EAAE,MAAM;YACtB,QAAQ;YACR,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;YAC5C,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC;YACzB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,GAAG;YACH,QAAQ,EAAE,IAAA,0BAAe,EAAC,SAAS,CAAC,KAAK,CAAC,EAAE;gBACxC,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM;gBACjC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACzD,CAAC;YACF,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;SAC1D,CAAC,CAAC;IACP,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,KAAK,CAAC,QAAQ,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACzB,SAAS;QACb,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,KAAK,MAAM,QAAQ,IAAI,IAAA,kCAAsB,EAAC,UAAU,CAAC,EAAE,CAAC;YACxD,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,gCAAoB,EAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,KAAK,EAAE,CAAC;YACR,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,IAAA,6BAAgB,EAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC"}

package/dist/segment/types.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Block model shared by both segmenters.
+ *
+ * A block is a contiguous run of a single representation. Blocks tile their representation
+ * completely: concatenating every block's `text` in order reproduces the canonical text exactly.
+ * That property is what makes the lossless reconstruction invariant testable, so it is maintained
+ * by construction rather than checked afterwards.
+ */
+import { type Representation } from '../types';
+/** Structural kind assigned during segmentation, before final classification. */
+export type BlockKind = 'paragraph' | 'quote' | 'reply_header' | 'forward_header' | 'quoted_header_block' | 'signature_candidate' | 'disclaimer_candidate' | 'list' | 'table' | 'separator' | 'image' | 'blank' | 'unknown';
+/** A single piece of evidence produced by a detector. Detectors never emit verdicts. */
+export interface Evidence {
+    /** Detector id, matching a pattern id or a selector id. */
+    id: string;
+    /** What the evidence suggests. */
+    kind: 'reply-anchor' | 'forward-anchor' | 'quote-container' | 'attribution' | 'separator' | 'signature' | 'disclaimer' | 'new-content' | 'hint';
+    /** How strongly, in [0, 1]. */
+    strength: number;
+}
+/**
+ * Inspectable feature vector.
+ *
+ * Every field here is dumped by `extract --explain`, and every field is a candidate input to a
+ * learned model later. Nothing is computed that cannot be shown to a human.
+ */
+export interface BlockFeatures {
+    /** Normalised position of the block start within the representation, in [0, 1]. */
+    position: number;
+    /** Index from the end, so that trailing structures are easy to express. */
+    indexFromEnd: number;
+    charCount: number;
+    lineCount: number;
+    /** Mean characters per non blank line. */
+    meanLineLength: number;
+    wordCount: number;
+    quoteDepth: number;
+    /** Nesting depth inside blockquote elements. HTML only. */
+    blockquoteDepth: number;
+    /** Number of HTML elements this block spans. HTML only. */
+    elementCount: number;
+    linkCount: number;
+    imageCount: number;
+    hasQuestionMark: boolean;
+    hasDate: boolean;
+    hasEmailAddress: boolean;
+    hasUrl: boolean;
+    hasPhoneNumber: boolean;
+    /** Lines that look like an embedded message header field. */
+    headerFieldLines: number;
+    /** Fraction of lines that are shorter than 40 characters, a signature-like shape. */
+    shortLineRatio: number;
+    /** Fraction of characters that are uppercase letters. */
+    upperCaseRatio: number;
+    /** Fraction of characters that are punctuation or symbols. */
+    punctuationRatio: number;
+    isAllBlank: boolean;
+    /** HTML tag names present, lowercased and deduplicated. */
+    tags: string[];
+    /** HTML class names present, lowercased and deduplicated. */
+    classes: string[];
+    /** True when the block has a top border, which several clients use as a reply separator. */
+    hasTopBorder: boolean;
+}
+export interface Block {
+    index: number;
+    kind: BlockKind;
+    representation: Representation;
+    /** MIME part this block came from, for example "1.2". */
+    mimePart: string;
+    /**
+     * Exact slice of the canonical representation, including quote prefixes and trailing blank
+     * lines. Concatenating this across all blocks reproduces the canonical text.
+     */
+    text: string;
+    /** Display and matching text: quote prefixes removed, surrounding blank lines trimmed. */
+    content: string;
+    /** Offsets into the canonical representation. */
+    start: number;
+    end: number;
+    /** Original HTML source for HTML blocks. Never regenerated. */
+    html?: string;
+    /** Stable child index path from the document root, for example "0/2/1". HTML only. */
+    domPath?: string;
+    features: BlockFeatures;
+    evidence: Evidence[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/segment/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/segment/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,iFAAiF;AACjF,MAAM,MAAM,SAAS,GACf,WAAW,GACX,OAAO,GACP,cAAc,GACd,gBAAgB,GAChB,qBAAqB,GACrB,qBAAqB,GACrB,sBAAsB,GACtB,MAAM,GACN,OAAO,GACP,WAAW,GACX,OAAO,GACP,OAAO,GACP,SAAS,CAAC;AAEhB,wFAAwF;AACxF,MAAM,WAAW,QAAQ;IACrB,2DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,kCAAkC;IAClC,IAAI,EAAE,cAAc,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,aAAa,GAAG,MAAM,CAAC;IAChJ,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC1B,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,eAAe,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,cAAc,EAAE,OAAO,CAAC;IACxB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,qFAAqF;IACrF,cAAc,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,2DAA2D;IAC3D,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,4FAA4F;IAC5F,YAAY,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,KAAK;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,cAAc,EAAE,cAAc,CAAC;IAC/B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,0FAA0F;IAC1F,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sFAAsF;IACtF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,aAAa,CAAC;IACxB,QAAQ,EAAE,QAAQ,EAAE,CAAC;CACxB"}

package/dist/segment/types.js

@@ -0,0 +1,11 @@
+"use strict";
+/**
+ * Block model shared by both segmenters.
+ *
+ * A block is a contiguous run of a single representation. Blocks tile their representation
+ * completely: concatenating every block's `text` in order reproduces the canonical text exactly.
+ * That property is what makes the lossless reconstruction invariant testable, so it is maintained
+ * by construction rather than checked afterwards.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/segment/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/segment/types.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG"}

package/dist/types.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Public contract for the extraction pipeline.
+ *
+ * Everything a caller needs to reason about the output lives here. The guiding rule for the whole
+ * project is asymmetric: losing genuine new content is far worse than showing redundant history,
+ * so every type in this file is designed so that content can be demoted to a lower visibility
+ * band but never dropped.
+ */
+/** Version of the deterministic extraction logic. Bump on any change that can alter a label. */
+export declare const EXTRACTOR_VERSION = "0.1.0";
+/** Version of the corpus processing pipeline (scanner, feature extraction, storage schema). */
+export declare const PROCESSING_VERSION = "0.1.0";
+/** Final block classification. */
+export type Label = 'NEW_CONTENT' | 'INLINE_QUOTE' | 'REPLY_HISTORY' | 'FORWARDED_CONTENT' | 'SIGNATURE' | 'DISCLAIMER' | 'UNKNOWN';
+export declare const LABELS: readonly Label[];
+/**
+ * How a client should present a segment.
+ *
+ * `hide` is only ever assigned to high confidence REPLY_HISTORY or FORWARDED_CONTENT that is
+ * backed by a structural anchor. `collapse` means recoverable with one interaction. Anything the
+ * pipeline is unsure about stays `show`.
+ */
+export type Visibility = 'show' | 'collapse' | 'hide';
+/** Which canonical representation a block came from. */
+export type Representation = 'text' | 'html';
+/**
+ * Traceability back to the original message. Every segment must carry one of these so a reviewer
+ * can always find the exact bytes the decision was made about.
+ */
+export interface SourceRef {
+    /** IMAP style MIME part path, for example "1.2". "TEXT" for a single part message root. */
+    mimePart: string;
+    representation: Representation;
+    /** Index of the block within its representation, in document order. */
+    blockIndex: number;
+    /** Character offsets into the decoded, canonicalised part content. */
+    start: number;
+    end: number;
+    /** HTML only. Stable child index path from the body element, for example "0/2/1". */
+    domPath?: string;
+}
+/**
+ * Machine readable justification codes. Closed vocabulary: a decision that cannot name its reason
+ * is not allowed to influence visibility.
+ */
+export type Reason = 'preceded_by_reply_header' | 'preceded_by_forward_header' | 'inside_quote_container' | 'final_contiguous_quote_region' | 'no_new_content_after_region' | 'new_content_after_region' | 'interleaved_with_new_content' | 'contains_old_message_headers' | 'quote_depth_positive' | 'no_anchor_for_quote' | 'gmail_quote_container' | 'gmail_attr_line' | 'gmail_signature_container' | 'outlook_reply_separator' | 'outlook_mso_border_separator' | 'owa_element_to_proof' | 'apple_mail_cite' | 'thunderbird_moz_cite' | 'generic_blockquote' | 'dash_dash_delimiter' | 'mobile_signature_phrase' | 'signature_position_and_length' | 'disclaimer_phrase' | 'matched_parent_message' | 'matched_thread_ancestor' | 'shingle_overlap_high' | 'no_history_available' | 'html_primary_representation' | 'text_primary_representation' | 'representations_disagree' | 'evidence_transferred_from_html' | 'evidence_transferred_from_text' | 'new_content_default' | 'ambiguous_kept_visible' | 'degraded_to_unknown';
+/** A classified run of content. */
+export interface Segment {
+    type: Label;
+    /** Plain text of the segment. Always present, always taken verbatim from the source. */
+    text: string;
+    /** Original HTML for HTML sourced segments. Never regenerated or rewritten. */
+    html?: string;
+    /** Calibrated in [0, 1]. */
+    confidence: number;
+    visibility: Visibility;
+    reasons: Reason[];
+    source: SourceRef;
+}
+/** Warning codes emitted during parsing or extraction. Never contains message content. */
+export type WarningCode = 'mime_parse_error' | 'mime_truncated' | 'missing_boundary' | 'max_nesting_exceeded' | 'max_parts_exceeded' | 'header_too_large' | 'unknown_charset' | 'charset_decode_replacement_chars' | 'transfer_encoding_error' | 'text_part_truncated' | 'html_parse_error' | 'no_body_parts' | 'non_contiguous_body' | 'extraction_degraded' | 'timeout';
+export interface Warning {
+    code: WarningCode;
+    /** Human readable detail. Must never include message body text or addresses. */
+    detail?: string;
+    mimePart?: string;
+}
+/** Result of extracting one message. */
+export interface ExtractionResult {
+    /** Segments the client should render, in document order. */
+    segments: Segment[];
+    /** Segments the client may hide or collapse. Retained in full so nothing is ever lost. */
+    hiddenSegments: Segment[];
+    /** Message level confidence, the weakest link across hide decisions. */
+    overallConfidence: number;
+    warnings: Warning[];
+    /** Always true. The pipeline never discards the source message. */
+    originalAvailable: true;
+    /** Which representation drove the final classification. */
+    primaryRepresentation: Representation | 'none';
+    extractorVersion: string;
+    /** True when thread history was available and used. */
+    historyUsed: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,gGAAgG;AAChG,eAAO,MAAM,iBAAiB,UAAU,CAAC;AAEzC,+FAA+F;AAC/F,eAAO,MAAM,kBAAkB,UAAU,CAAC;AAE1C,kCAAkC;AAClC,MAAM,MAAM,KAAK,GAAG,aAAa,GAAG,cAAc,GAAG,eAAe,GAAG,mBAAmB,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;AAEpI,eAAO,MAAM,MAAM,EAAE,SAAS,KAAK,EAAyH,CAAC;AAE7J;;;;;;GAMG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,CAAC;AAE7C;;;GAGG;AACH,MAAM,WAAW,SAAS;IACtB,2FAA2F;IAC3F,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,cAAc,CAAC;IAC/B,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,MAAM,MAAM,GAEZ,0BAA0B,GAC1B,4BAA4B,GAC5B,wBAAwB,GACxB,+BAA+B,GAC/B,6BAA6B,GAC7B,0BAA0B,GAC1B,8BAA8B,GAC9B,8BAA8B,GAC9B,sBAAsB,GACtB,qBAAqB,GAErB,uBAAuB,GACvB,iBAAiB,GACjB,2BAA2B,GAC3B,yBAAyB,GACzB,8BAA8B,GAC9B,sBAAsB,GACtB,iBAAiB,GACjB,sBAAsB,GACtB,oBAAoB,GAEpB,qBAAqB,GACrB,yBAAyB,GACzB,+BAA+B,GAC/B,mBAAmB,GAEnB,wBAAwB,GACxB,yBAAyB,GACzB,sBAAsB,GACtB,sBAAsB,GAEtB,6BAA6B,GAC7B,6BAA6B,GAC7B,0BAA0B,GAC1B,gCAAgC,GAChC,gCAAgC,GAEhC,qBAAqB,GACrB,wBAAwB,GACxB,qBAAqB,CAAC;AAE5B,mCAAmC;AACnC,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,KAAK,CAAC;IACZ,wFAAwF;IACxF,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;IACvB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,SAAS,CAAC;CACrB;AAED,0FAA0F;AAC1F,MAAM,MAAM,WAAW,GACjB,kBAAkB,GAClB,gBAAgB,GAChB,kBAAkB,GAClB,sBAAsB,GACtB,oBAAoB,GACpB,kBAAkB,GAClB,iBAAiB,GACjB,kCAAkC,GAClC,yBAAyB,GACzB,qBAAqB,GACrB,kBAAkB,GAClB,eAAe,GACf,qBAAqB,GACrB,qBAAqB,GACrB,SAAS,CAAC;AAEhB,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,WAAW,CAAC;IAClB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC7B,4DAA4D;IAC5D,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,0FAA0F;IAC1F,cAAc,EAAE,OAAO,EAAE,CAAC;IAC1B,wEAAwE;IACxE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,mEAAmE;IACnE,iBAAiB,EAAE,IAAI,CAAC;IACxB,2DAA2D;IAC3D,qBAAqB,EAAE,cAAc,GAAG,MAAM,CAAC;IAC/C,gBAAgB,EAAE,MAAM,CAAC;IACzB,uDAAuD;IACvD,WAAW,EAAE,OAAO,CAAC;CACxB"}

package/dist/types.js

@@ -0,0 +1,17 @@
+"use strict";
+/**
+ * Public contract for the extraction pipeline.
+ *
+ * Everything a caller needs to reason about the output lives here. The guiding rule for the whole
+ * project is asymmetric: losing genuine new content is far worse than showing redundant history,
+ * so every type in this file is designed so that content can be demoted to a lower visibility
+ * band but never dropped.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LABELS = exports.PROCESSING_VERSION = exports.EXTRACTOR_VERSION = void 0;
+/** Version of the deterministic extraction logic. Bump on any change that can alter a label. */
+exports.EXTRACTOR_VERSION = '0.1.0';
+/** Version of the corpus processing pipeline (scanner, feature extraction, storage schema). */
+exports.PROCESSING_VERSION = '0.1.0';
+exports.LABELS = ['NEW_CONTENT', 'INLINE_QUOTE', 'REPLY_HISTORY', 'FORWARDED_CONTENT', 'SIGNATURE', 'DISCLAIMER', 'UNKNOWN'];
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;AAEH,gGAAgG;AACnF,QAAA,iBAAiB,GAAG,OAAO,CAAC;AAEzC,+FAA+F;AAClF,QAAA,kBAAkB,GAAG,OAAO,CAAC;AAK7B,QAAA,MAAM,GAAqB,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,mBAAmB,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,CAAU,CAAC"}

package/dist/util/hash.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Hashing helpers.
+ *
+ * Two jobs: content addressing for exact duplicate detection, and privacy preserving identifiers so
+ * that the corpus database can group by sender or subject without storing either. Salted hashes use
+ * a per database salt, so a leaked database cannot be dictionary attacked against a known address
+ * list without also leaking the salt.
+ */
+export declare function sha256(buf: Buffer | string): string;
+/** Short content hash, enough to group duplicates within a single corpus. */
+export declare function shortHash(buf: Buffer | string): string;
+/** Keyed hash for values that must be groupable but never recoverable. */
+export declare function saltedHash(salt: string, value: string): string;
+/** Generate a random salt for a new corpus database. */
+export declare function newSalt(): string;
+/**
+ * Deterministic 32 bit selector in [0, 1) derived from a key and a seed.
+ *
+ * Used for reproducible sampling: the same file always lands in the same position for a given seed,
+ * without needing to hold the corpus in memory or shuffle it.
+ */
+export declare function deterministicUnit(seed: string, key: string): number;
+/** FNV-1a, 32 bit. Fast and good enough for shingling and simhash token weights. */
+export declare function fnv1a32(value: string): number;
+/** Second, independent 32 bit hash so that two of them can form a 64 bit value. */
+export declare function fnv1a32b(value: string): number;
+/**
+ * 64 bit SimHash over whitespace separated tokens, returned as a 16 character hex string.
+ *
+ * Near duplicate detection matters for split hygiene: a template that appears thousands of times
+ * must not be spread across train, validation and test. Hamming distance on this value is a cheap
+ * first pass filter before any expensive comparison.
+ */
+export declare function simhash64(text: string): string;
+/**
+ * Default Hamming distance below which two bodies are treated as near duplicates.
+ *
+ * Measured against a 300 character templated invoice mail: changing one identifier gives distance
+ * 4, changing two fields gives 7, adding a sentence gives 6, and an unrelated message gives 30.
+ * Eight therefore separates template variants from distinct content with a comfortable margin.
+ *
+ * Short bodies are noisier, because each token carries more of the signal. A one token change in a
+ * 14 token message already gives distance 8. Callers comparing very short texts should either
+ * tighten this or fall back to exact comparison.
+ */
+export declare const NEAR_DUPLICATE_THRESHOLD = 8;
+/** Hamming distance between two hex SimHash values. Returns 64 when either is malformed. */
+export declare function simhashDistance(a: string, b: string): number;
+//# sourceMappingURL=hash.d.ts.map

package/dist/util/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../src/util/hash.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEnD;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAEtD;AAED,0EAA0E;AAC1E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE9D;AAED,wDAAwD;AACxD,wBAAgB,OAAO,IAAI,MAAM,CAEhC;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGnE;AAED,oFAAoF;AACpF,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED,mFAAmF;AACnF,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS9C;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAkC9C;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAE1C,4FAA4F;AAC5F,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAkB5D"}

package/dist/util/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/util/hash.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;AAIH,wBAEC;AAGD,8BAEC;AAGD,gCAEC;AAGD,0BAEC;AAQD,8CAGC;AAGD,0BASC;AAGD,4BASC;AASD,8BAkCC;AAgBD,0CAkBC;AAnID,8DAAiC;AAEjC,SAAgB,MAAM,CAAC,GAAoB;IACvC,OAAO,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACjE,CAAC;AAED,6EAA6E;AAC7E,SAAgB,SAAS,CAAC,GAAoB;IAC1C,OAAO,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,0EAA0E;AAC1E,SAAgB,UAAU,CAAC,IAAY,EAAE,KAAa;IAClD,OAAO,qBAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtF,CAAC;AAED,wDAAwD;AACxD,SAAgB,OAAO;IACnB,OAAO,qBAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,IAAY,EAAE,GAAW;IACvD,MAAM,MAAM,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;IACzF,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;AAChD,CAAC;AAED,oFAAoF;AACpF,SAAgB,OAAO,CAAC,KAAa;IACjC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,mFAAmF;AACnF,SAAgB,QAAQ,CAAC,KAAa;IAClC,IAAI,IAAI,GAAG,UAAU,GAAG,UAAU,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACtB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,SAAS,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI;SACd,WAAW,EAAE;SACb,KAAK,CAAC,iBAAiB,CAAC;SACxB,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACzE,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,IAAI,MAAM,GAAG,EAAE,EAAE,MAAM,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC;YAC/B,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;IACnC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED;;;;;;;;;;GAUG;AACU,QAAA,wBAAwB,GAAG,CAAC,CAAC;AAE1C,4FAA4F;AAC5F,SAAgB,eAAe,CAAC,CAAS,EAAE,CAAS;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,EAAE,IAAI,CAAC,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACrC,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAW,EAAE,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAW,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACd,CAAC;QACD,IAAI,GAAG,GAAG,IAAI,GAAG,KAAK,CAAC;QACvB,OAAO,GAAG,EAAE,CAAC;YACT,QAAQ,IAAI,GAAG,GAAG,CAAC,CAAC;YACpB,GAAG,KAAK,CAAC,CAAC;QACd,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC"}

package/dist/util/lang.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Language estimation.
+ *
+ * Deliberately small and dependency free. This is an estimate used for stratified sampling and for
+ * reporting metrics per language, never for a classification decision. A wrong guess costs a
+ * slightly skewed sample, nothing more, which is why a stopword scorer is preferred over pulling in
+ * a model.
+ *
+ * Two stages: script detection first, because a Cyrillic or CJK body is unambiguous and cheap to
+ * spot, then stopword frequency for Latin script languages.
+ */
+export type LanguageCode = 'en' | 'et' | 'de' | 'fr' | 'es' | 'it' | 'pt' | 'nl' | 'sv' | 'fi' | 'pl' | 'ru' | 'zh' | 'ja' | 'ko' | 'ar' | 'el' | 'he' | 'unknown';
+export interface LanguageEstimate {
+    lang: LanguageCode;
+    /** Share of sampled tokens that matched the winning language's stopword list. */
+    score: number;
+}
+/**
+ * Estimate the language of a body of text.
+ *
+ * @param text plain text. Only the first 20k characters are inspected, which is ample and keeps the
+ *   cost bounded on newsletters and digests.
+ */
+export declare function estimateLanguage(text: string): LanguageEstimate;
+//# sourceMappingURL=lang.d.ts.map

package/dist/util/lang.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lang.d.ts","sourceRoot":"","sources":["../../src/util/lang.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,MAAM,YAAY,GACpB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,SAAS,CAAC;AAE5I,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,YAAY,CAAC;IACnB,iFAAiF;IACjF,KAAK,EAAE,MAAM,CAAC;CACjB;AA8ED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CA8C/D"}