@postalsys/email-content 0.0.1

package/dist/corpus/analyze.js

@@ -0,0 +1,214 @@
+"use strict";
+/**
+ * Corpus feature extraction.
+ *
+ * Turns a parsed message into the flat, privacy safe record stored in the corpus database. The hard
+ * rule here is that nothing derived from body text, subjects or addresses may leave this function
+ * in readable form. Subjects and addresses become salted hashes, sender domains are kept because
+ * they are needed for stratification, and body text is reduced to counts, ratios and a SimHash.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyzeMessage = analyzeMessage;
+const html_1 = require("../canonical/html");
+const text_1 = require("../canonical/text");
+const client_1 = require("../detect/client");
+const patterns_1 = require("../detect/patterns");
+const traffic_1 = require("../detect/traffic");
+const hash_1 = require("../util/hash");
+const lang_1 = require("../util/lang");
+const REPLY_SUBJECT = /^\s*(re|aw|sv|vs|vast|antw|res|odp|ответ)\s*(\[\d+\])?\s*:/i;
+const FORWARD_SUBJECT = /^\s*(fwd?|tr|wg|vs|vl|doorst|пересылка)\s*:/i;
+function pickParts(message) {
+    const plain = [];
+    const html = [];
+    let attachments = 0;
+    for (const part of message.parts) {
+        if (part.multipart) {
+            continue;
+        }
+        const isText = part.contentType === 'text/plain' || part.contentType === 'text/html';
+        if (isText && part.disposition !== 'attachment') {
+            if (part.contentType === 'text/plain') {
+                plain.push(part);
+            }
+            else {
+                html.push(part);
+            }
+            continue;
+        }
+        if (part.rfc822) {
+            continue;
+        }
+        attachments++;
+    }
+    return { plain, html, attachments };
+}
+function distinct(values) {
+    const set = new Set();
+    for (const value of values) {
+        if (value) {
+            set.add(value.toLowerCase());
+        }
+    }
+    return [...set].sort().join(',');
+}
+function extractDomain(address) {
+    if (!address) {
+        return null;
+    }
+    const match = address.match(/@([A-Za-z0-9.-]+\.[A-Za-z]{2,})/);
+    return match?.[1] ? match[1].toLowerCase() : null;
+}
+function extractAddress(header) {
+    if (!header) {
+        return null;
+    }
+    const angled = header.match(/<([^<>@\s]{1,64}@[^<>\s]{1,255})>/);
+    if (angled?.[1]) {
+        return angled[1].toLowerCase();
+    }
+    const bare = header.match(/([^<>\s,;"]{1,64}@[^<>\s,;"]{1,255})/);
+    return bare?.[1] ? bare[1].toLowerCase() : null;
+}
+/** Which line level markers fired anywhere in the text. Ids only, no matched content. */
+function detectMarkers(lines) {
+    const patterns = (0, patterns_1.loadPatterns)();
+    const fired = new Set();
+    // Cap the scan. Reply markers cluster near the top and bottom of a body, and a digest with tens
+    // of thousands of lines would otherwise dominate the scan cost.
+    const limit = 3000;
+    const scan = lines.length > limit ? [...lines.slice(0, limit / 2), ...lines.slice(-limit / 2)] : lines;
+    for (const line of scan) {
+        if (line.length > 500) {
+            continue;
+        }
+        for (const pattern of patterns.replyHeaders) {
+            if (pattern.regex.test(line)) {
+                fired.add(`reply:${pattern.id}`);
+            }
+        }
+        for (const pattern of patterns.forwardHeaders) {
+            if (pattern.regex.test(line)) {
+                fired.add(`forward:${pattern.id}`);
+            }
+        }
+        for (const pattern of patterns.quotedHeaderFields) {
+            if (pattern.regex.test(line)) {
+                fired.add(`field:${pattern.id}`);
+            }
+        }
+        for (const pattern of patterns.signatureDelimiters) {
+            if (pattern.regex.test(line)) {
+                fired.add(`sigdelim:${pattern.id}`);
+            }
+        }
+        for (const pattern of patterns.signaturePhrases) {
+            if (pattern.regex.test(line)) {
+                fired.add(`sig:${pattern.id}`);
+            }
+        }
+    }
+    return [...fired].sort();
+}
+function detectDisclaimers(text) {
+    const patterns = (0, patterns_1.loadPatterns)();
+    // Disclaimers sit at the end. Scanning the tail keeps the cost bounded and avoids matching a
+    // discussion that happens to mention confidentiality in the middle of a thread.
+    const tail = text.length > 4000 ? text.slice(-4000) : text;
+    const fired = [];
+    for (const pattern of patterns.disclaimerPhrases) {
+        if (pattern.regex.test(tail)) {
+            fired.push(`disclaimer:${pattern.id}`);
+        }
+    }
+    return fired;
+}
+/**
+ * Build the corpus record for one parsed message.
+ *
+ * @param message parsed MIME structure
+ * @param salt per database salt for the privacy preserving hashes
+ */
+function analyzeMessage(message, salt) {
+    const { plain, html, attachments } = pickParts(message);
+    const headers = message.headers;
+    const plainText = plain
+        .map(part => part.decoded?.text || '')
+        .filter(Boolean)
+        .join('\n');
+    const htmlSource = html
+        .map(part => part.decoded?.text || '')
+        .filter(Boolean)
+        .join('\n');
+    const canonical = (0, text_1.canonicaliseText)(plainText);
+    const quoting = (0, text_1.profileQuoting)(canonical.lines);
+    const hints = htmlSource ? (0, html_1.detectHtmlHints)(htmlSource) : null;
+    // Language and markers come from whichever representation actually carries the content.
+    const htmlProjection = htmlSource && !plainText ? (0, html_1.stripTagsToText)(htmlSource) : '';
+    const analysisText = plainText || htmlProjection;
+    const markers = detectMarkers(canonical.lines.map(line => line.content));
+    if (htmlProjection) {
+        markers.push(...detectMarkers(htmlProjection.split('\n')));
+    }
+    markers.push(...detectDisclaimers(analysisText));
+    const bodyKind = plainText && htmlSource ? 'both' : plainText ? 'plain-only' : htmlSource ? 'html-only' : 'none';
+    const subject = headers.subject || '';
+    const subjectPrefix = !subject
+        ? 'none'
+        : REPLY_SUBJECT.test(subject)
+            ? 'reply'
+            : FORWARD_SUBJECT.test(subject)
+                ? 'forward'
+                : 'plain';
+    const rootPart = message.partsById.get(message.rootId);
+    const boundary = rootPart?.contentTypeParams.boundary || null;
+    const fingerprint = (0, client_1.fingerprintClient)(headers, htmlSource || null, boundary);
+    const traffic = (0, traffic_1.classifyTraffic)(headers, message.rootHeaders);
+    const language = (0, lang_1.estimateLanguage)(analysisText);
+    const hasQuoteInPlain = quoting.quotedLines > 0 || markers.some(id => id.startsWith('reply:'));
+    const hasQuoteInHtml = Boolean(hints && (hints.hasBlockquote || hints.ids.some(id => id.includes('quote') || id.includes('Rply') || id.includes('separator'))));
+    const fromAddress = extractAddress(headers.from);
+    return {
+        messageId: headers.messageId,
+        inReplyTo: headers.inReplyTo,
+        referencesCount: headers.references.length,
+        referencesRoot: headers.references[0] ?? null,
+        dateMs: headers.dateMs,
+        fromDomain: extractDomain(headers.from),
+        fromHash: fromAddress ? (0, hash_1.saltedHash)(salt, fromAddress) : null,
+        subjectHash: subject ? (0, hash_1.saltedHash)(salt, subject.trim().toLowerCase().replace(REPLY_SUBJECT, '').replace(FORWARD_SUBJECT, '').trim()) : null,
+        subjectPrefix,
+        subjectLength: subject.length,
+        structure: message.structure,
+        bodyKind,
+        charsets: distinct([...plain, ...html].map(part => part.decoded?.charsetUsed ?? part.charset)),
+        encodings: distinct(message.parts.filter(part => !part.multipart).map(part => part.encoding)),
+        partCount: message.parts.length,
+        attachmentCount: attachments,
+        hasRfc822: message.parts.some(part => part.rfc822),
+        textChars: plainText.length,
+        htmlChars: htmlSource.length,
+        flowed: plain.some(part => part.flowed),
+        client: fingerprint.client,
+        clientSource: fingerprint.source,
+        traffic: traffic.traffic,
+        trafficSignals: traffic.signals.join(','),
+        isReply: Boolean(headers.inReplyTo) || subjectPrefix === 'reply',
+        isForward: subjectPrefix === 'forward' || markers.some(id => id.startsWith('forward:')),
+        quoteDepthMax: quoting.maxDepth,
+        quotedRatio: Number(quoting.quotedRatio.toFixed(4)),
+        replyStyle: plainText ? quoting.style : 'unknown',
+        markers: markers.sort().join(','),
+        htmlHints: hints ? hints.ids.join(',') : '',
+        blockquoteCount: hints?.blockquoteCount ?? 0,
+        tableCount: hints?.tableCount ?? 0,
+        hasImage: hints?.hasImage ?? false,
+        suspectMalformedHtml: hints?.suspectMalformed ?? false,
+        lang: language.lang,
+        langScore: Number(language.score.toFixed(3)),
+        bodySimhash: (0, hash_1.simhash64)(analysisText),
+        warnings: [...new Set(message.warnings.map(warning => warning.code))].sort().join(','),
+        quoteMarkingDisagrees: bodyKind === 'both' && hasQuoteInPlain !== hasQuoteInHtml
+    };
+}
+//# sourceMappingURL=analyze.js.map

package/dist/corpus/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../src/corpus/analyze.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AAkLH,wCA6FC;AA7QD,4CAAqE;AACrE,4CAAsF;AACtF,6CAAoE;AACpE,iDAAkD;AAClD,+CAAuE;AAEvE,uCAAqD;AACrD,uCAAmE;AAiDnE,MAAM,aAAa,GAAG,6DAA6D,CAAC;AACpF,MAAM,eAAe,GAAG,8CAA8C,CAAC;AAEvE,SAAS,SAAS,CAAC,OAAsB;IACrC,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,KAAK,YAAY,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,CAAC;QACrF,IAAI,MAAM,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;gBACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;YACD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,WAAW,EAAE,CAAC;IAClB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC;AAED,SAAS,QAAQ,CAAC,MAAkC;IAChD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,aAAa,CAAC,OAAsB;IACzC,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC/D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,cAAc,CAAC,MAAqB;IACzC,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACjE,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACd,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACnC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAClE,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,yFAAyF;AACzF,SAAS,aAAa,CAAC,KAAwB;IAC3C,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,gGAAgG;IAChG,gEAAgE;IAChE,MAAM,KAAK,GAAG,IAAI,CAAC;IACnB,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEvG,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACpB,SAAS;QACb,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,kBAAkB,EAAE,CAAC;YAChD,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,CAAC;QACL,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACnC,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,6FAA6F;IAC7F,gFAAgF;IAChF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,OAAsB,EAAE,IAAY;IAC/D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAEhC,MAAM,SAAS,GAAG,KAAK;SAClB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;SACrC,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,MAAM,UAAU,GAAG,IAAI;SAClB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;SACrC,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhB,MAAM,SAAS,GAAG,IAAA,uBAAgB,EAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAA,qBAAc,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,sBAAe,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,wFAAwF;IACxF,MAAM,cAAc,GAAG,UAAU,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAA,sBAAe,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnF,MAAM,YAAY,GAAG,SAAS,IAAI,cAAc,CAAC;IAEjD,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACzE,IAAI,cAAc,EAAE,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;IAEjD,MAAM,QAAQ,GAAa,SAAS,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;IAE3H,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,aAAa,GAAmC,CAAC,OAAO;QAC1D,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;YAC3B,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC7B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,OAAO,CAAC;IAElB,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC9D,MAAM,WAAW,GAAG,IAAA,0BAAiB,EAAC,OAAO,EAAE,UAAU,IAAI,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAA,yBAAe,EAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,IAAA,uBAAgB,EAAC,YAAY,CAAC,CAAC;IAEhD,MAAM,eAAe,GAAG,OAAO,CAAC,WAAW,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/F,MAAM,cAAc,GAAG,OAAO,CAC1B,KAAK,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAClI,CAAC;IAEF,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,eAAe,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM;QAC1C,cAAc,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI;QAC7C,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC;QACvC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,IAAA,iBAAU,EAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI;QAC5D,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,IAAA,iBAAU,EAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QAC3I,aAAa;QACb,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ;QACR,QAAQ,EAAE,QAAQ,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9F,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7F,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;QAC/B,eAAe,EAAE,WAAW;QAC5B,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;QAClD,SAAS,EAAE,SAAS,CAAC,MAAM;QAC3B,SAAS,EAAE,UAAU,CAAC,MAAM;QAC5B,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;QACvC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,YAAY,EAAE,WAAW,CAAC,MAAM;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,aAAa,KAAK,OAAO;QAChE,SAAS,EAAE,aAAa,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACvF,aAAa,EAAE,OAAO,CAAC,QAAQ;QAC/B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnD,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACjD,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;QACjC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE;QAC3C,eAAe,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;QAC5C,UAAU,EAAE,KAAK,EAAE,UAAU,IAAI,CAAC;QAClC,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,KAAK;QAClC,oBAAoB,EAAE,KAAK,EAAE,gBAAgB,IAAI,KAAK;QACtD,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5C,WAAW,EAAE,IAAA,gBAAS,EAAC,YAAY,CAAC;QACpC,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;QACtF,qBAAqB,EAAE,QAAQ,KAAK,MAAM,IAAI,eAAe,KAAK,cAAc;KACnF,CAAC;AACN,CAAC"}

package/dist/corpus/migrations/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Forward only schema migrations.
+ *
+ * Applied in order, tracked with SQLite's user_version pragma. Migrations never rewrite history:
+ * to change a column, add a new migration. A corpus scan takes minutes and can be re-run, but
+ * annotations cannot be regenerated, so the annotation tables must only ever be extended.
+ */
+export interface Migration {
+    version: number;
+    name: string;
+    sql: string;
+}
+export declare const MIGRATIONS: readonly Migration[];
+//# sourceMappingURL=index.d.ts.map

package/dist/corpus/migrations/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/corpus/migrations/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,SAAS;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,UAAU,EAAE,SAAS,SAAS,EAkJ1C,CAAC"}

package/dist/corpus/migrations/index.js

@@ -0,0 +1,158 @@
+"use strict";
+/**
+ * Forward only schema migrations.
+ *
+ * Applied in order, tracked with SQLite's user_version pragma. Migrations never rewrite history:
+ * to change a column, add a new migration. A corpus scan takes minutes and can be re-run, but
+ * annotations cannot be regenerated, so the annotation tables must only ever be extended.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MIGRATIONS = void 0;
+exports.MIGRATIONS = [
+    {
+        version: 1,
+        name: 'corpus-core',
+        sql: `
+CREATE TABLE meta (
+    key   TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+
+-- One row per scan invocation, so that results can be attributed to a code version.
+CREATE TABLE runs (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    root                TEXT NOT NULL,
+    started_at          INTEGER NOT NULL,
+    finished_at         INTEGER,
+    status              TEXT NOT NULL DEFAULT 'running',
+    processing_version  TEXT NOT NULL,
+    extractor_version   TEXT NOT NULL,
+    options_json        TEXT NOT NULL DEFAULT '{}',
+    files_seen          INTEGER NOT NULL DEFAULT 0,
+    files_processed     INTEGER NOT NULL DEFAULT 0,
+    files_failed        INTEGER NOT NULL DEFAULT 0,
+    files_skipped       INTEGER NOT NULL DEFAULT 0
+);
+
+-- One row per .eml file. Path is relative to the run root so a corpus can be moved.
+CREATE TABLE files (
+    path                TEXT PRIMARY KEY,
+    size                INTEGER NOT NULL,
+    mtime               INTEGER NOT NULL,
+    sha256              TEXT,
+    status              TEXT NOT NULL,
+    error_code          TEXT,
+    error_detail        TEXT,
+    duration_ms         INTEGER,
+    scanned_at          INTEGER NOT NULL,
+    run_id              INTEGER,
+    processing_version  TEXT NOT NULL
+);
+CREATE INDEX files_sha256   ON files(sha256);
+CREATE INDEX files_status   ON files(status);
+
+-- Derived, privacy safe metadata. No subjects, no addresses, no body text.
+CREATE TABLE messages (
+    path                    TEXT PRIMARY KEY REFERENCES files(path) ON DELETE CASCADE,
+    message_id              TEXT,
+    in_reply_to             TEXT,
+    references_count        INTEGER NOT NULL DEFAULT 0,
+    references_root         TEXT,
+    date_ms                 INTEGER,
+    from_domain             TEXT,
+    from_hash               TEXT,
+    subject_hash            TEXT,
+    subject_prefix          TEXT,
+    subject_length          INTEGER NOT NULL DEFAULT 0,
+    structure               TEXT,
+    body_kind               TEXT,
+    charsets                TEXT,
+    encodings               TEXT,
+    part_count              INTEGER NOT NULL DEFAULT 0,
+    attachment_count        INTEGER NOT NULL DEFAULT 0,
+    has_rfc822              INTEGER NOT NULL DEFAULT 0,
+    text_chars              INTEGER NOT NULL DEFAULT 0,
+    html_chars              INTEGER NOT NULL DEFAULT 0,
+    flowed                  INTEGER NOT NULL DEFAULT 0,
+    client                  TEXT,
+    client_source           TEXT,
+    traffic                 TEXT,
+    traffic_signals         TEXT,
+    is_reply                INTEGER NOT NULL DEFAULT 0,
+    is_forward              INTEGER NOT NULL DEFAULT 0,
+    quote_depth_max         INTEGER NOT NULL DEFAULT 0,
+    quoted_ratio            REAL NOT NULL DEFAULT 0,
+    reply_style             TEXT,
+    markers                 TEXT,
+    html_hints              TEXT,
+    blockquote_count        INTEGER NOT NULL DEFAULT 0,
+    table_count             INTEGER NOT NULL DEFAULT 0,
+    has_image               INTEGER NOT NULL DEFAULT 0,
+    suspect_malformed_html  INTEGER NOT NULL DEFAULT 0,
+    lang                    TEXT,
+    lang_score              REAL NOT NULL DEFAULT 0,
+    body_simhash            TEXT,
+    warnings                TEXT,
+    quote_marking_disagrees INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX messages_message_id ON messages(message_id);
+CREATE INDEX messages_in_reply_to ON messages(in_reply_to);
+CREATE INDEX messages_client      ON messages(client);
+CREATE INDEX messages_traffic     ON messages(traffic);
+CREATE INDEX messages_simhash     ON messages(body_simhash);
+CREATE INDEX messages_subject     ON messages(subject_hash);
+`
+    },
+    {
+        version: 2,
+        name: 'annotations',
+        sql: `
+-- One row per annotation pass over a set of messages, so that labels can always be attributed to
+-- a labeller, a model and a prompt version.
+CREATE TABLE annotation_runs (
+    id                 INTEGER PRIMARY KEY AUTOINCREMENT,
+    source             TEXT NOT NULL,          -- 'llm' | 'human' | 'weak' | 'distant'
+    model              TEXT,                   -- model id for 'llm'
+    prompt_version     TEXT,
+    extractor_version  TEXT NOT NULL,
+    started_at         INTEGER NOT NULL,
+    finished_at        INTEGER,
+    messages           INTEGER NOT NULL DEFAULT 0,
+    blocks             INTEGER NOT NULL DEFAULT 0,
+    failures           INTEGER NOT NULL DEFAULT 0,
+    prompt_tokens      INTEGER NOT NULL DEFAULT 0,
+    completion_tokens  INTEGER NOT NULL DEFAULT 0,
+    notes              TEXT
+);
+
+-- Block level labels. Deliberately not unique on (path, block_index): the same block may be
+-- labelled by several sources, and comparing them is the point.
+CREATE TABLE annotations (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id          INTEGER NOT NULL REFERENCES annotation_runs(id) ON DELETE CASCADE,
+    path            TEXT NOT NULL,
+    representation  TEXT NOT NULL,
+    block_index     INTEGER NOT NULL,
+    mime_part       TEXT,
+    block_start     INTEGER,
+    block_end       INTEGER,
+    char_count      INTEGER NOT NULL DEFAULT 0,
+    source          TEXT NOT NULL,
+    label           TEXT NOT NULL,
+    confidence      REAL NOT NULL DEFAULT 0,
+    -- What the deterministic baseline said about the same block, captured at annotation time so a
+    -- later rule change cannot silently rewrite history.
+    baseline_label  TEXT,
+    baseline_conf   REAL,
+    baseline_reasons TEXT,
+    agrees          INTEGER NOT NULL DEFAULT 0,
+    notes           TEXT
+);
+CREATE INDEX annotations_path   ON annotations(path);
+CREATE INDEX annotations_run    ON annotations(run_id);
+CREATE INDEX annotations_source ON annotations(source);
+CREATE INDEX annotations_agree  ON annotations(agrees);
+`
+    }
+];
+//# sourceMappingURL=index.js.map

package/dist/corpus/migrations/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/corpus/migrations/index.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;AAQU,QAAA,UAAU,GAAyB;IAC5C;QACI,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,aAAa;QACnB,GAAG,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyFZ;KACI;IACD;QACI,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,aAAa;QACnB,GAAG,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8CZ;KACI;CACJ,CAAC"}

package/dist/corpus/sample.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Stratified sampling over the corpus.
+ *
+ * Uniform random sampling of this corpus would produce a set that is almost entirely Gmail top
+ * posts in English, because that is what the corpus mostly is. The structures that actually break
+ * an extractor, interleaved replies, Outlook's unmarked history, malformed HTML, plain text only
+ * mail, are each a percent or two, and a random sample of a few hundred messages would contain a
+ * handful of each at best.
+ *
+ * So strata are defined by the structural properties that change how extraction behaves, each gets
+ * a quota, and a random control slice is included alongside so that unbiased numbers remain
+ * available. Selection within a stratum is deterministic: the same seed always picks the same
+ * messages, regardless of database order.
+ */
+import { type CorpusStore } from './store';
+export interface Stratum {
+    id: string;
+    /** SQL predicate over the messages table, aliased m, joined to files as f. */
+    predicate: string;
+    /** Share of the total sample to allocate. Normalised across strata. */
+    weight: number;
+    description: string;
+}
+/**
+ * The strata that matter, with the reason each one exists.
+ *
+ * Weights are not proportional to prevalence. They are proportional to how much is learned per
+ * message, which for a rare structure is a great deal more.
+ */
+export declare const DEFAULT_STRATA: readonly Stratum[];
+export interface SampleOptions {
+    /** Total messages wanted. */
+    count: number;
+    seed: string;
+    /** Restrict to a traffic class, or 'all'. */
+    traffic?: string;
+    /** Only messages that look like replies. */
+    repliesOnly?: boolean;
+    /** Exclude these paths, for example ones already annotated. */
+    exclude?: ReadonlySet<string>;
+    strata?: readonly Stratum[];
+}
+export interface SampleRow {
+    path: string;
+    stratum: string;
+}
+export interface SampleResult {
+    rows: SampleRow[];
+    /** How many were taken from each stratum, and how many were available. */
+    coverage: {
+        id: string;
+        requested: number;
+        taken: number;
+        available: number;
+        description: string;
+    }[];
+}
+/**
+ * Draw a stratified sample.
+ *
+ * Strata overlap by design: a message can be an Outlook interleaved reply in Estonian. It is
+ * assigned to the first stratum that claims it, so the rarer strata listed earlier fill first, and
+ * no message appears twice.
+ */
+export declare function stratifiedSample(store: CorpusStore, options: SampleOptions): SampleResult;
+//# sourceMappingURL=sample.d.ts.map

package/dist/corpus/sample.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample.d.ts","sourceRoot":"","sources":["../../src/corpus/sample.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,WAAW,OAAO;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,SAAS,OAAO,EAyB5C,CAAC;AAEF,MAAM,WAAW,aAAa;IAC1B,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,SAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACxG;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,GAAG,YAAY,CA+CzF"}

package/dist/corpus/sample.js

@@ -0,0 +1,98 @@
+"use strict";
+/**
+ * Stratified sampling over the corpus.
+ *
+ * Uniform random sampling of this corpus would produce a set that is almost entirely Gmail top
+ * posts in English, because that is what the corpus mostly is. The structures that actually break
+ * an extractor, interleaved replies, Outlook's unmarked history, malformed HTML, plain text only
+ * mail, are each a percent or two, and a random sample of a few hundred messages would contain a
+ * handful of each at best.
+ *
+ * So strata are defined by the structural properties that change how extraction behaves, each gets
+ * a quota, and a random control slice is included alongside so that unbiased numbers remain
+ * available. Selection within a stratum is deterministic: the same seed always picks the same
+ * messages, regardless of database order.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_STRATA = void 0;
+exports.stratifiedSample = stratifiedSample;
+const hash_1 = require("../util/hash");
+/**
+ * The strata that matter, with the reason each one exists.
+ *
+ * Weights are not proportional to prevalence. They are proportional to how much is learned per
+ * message, which for a rare structure is a great deal more.
+ */
+exports.DEFAULT_STRATA = [
+    { id: 'interleaved', predicate: "m.reply_style = 'interleaved'", weight: 3, description: 'inline question and answer, the highest cost failure mode' },
+    { id: 'bottom-post', predicate: "m.reply_style = 'bottom'", weight: 2, description: 'answer below the quote' },
+    { id: 'top-and-trailing', predicate: "m.reply_style = 'top-and-trailing'", weight: 2, description: 'top post with something after the quote' },
+    { id: 'plain-only', predicate: "m.body_kind = 'plain-only'", weight: 2, description: 'no HTML to fall back on' },
+    { id: 'html-only', predicate: "m.body_kind = 'html-only'", weight: 2, description: 'no plain text to fall back on' },
+    { id: 'representations-disagree', predicate: 'm.quote_marking_disagrees = 1', weight: 2, description: 'plain and HTML tell different stories' },
+    {
+        id: 'outlook',
+        predicate: "m.client IN ('outlook-desktop', 'outlook-web', 'outlook-mobile')",
+        weight: 3,
+        description: 'unmarked history, separators, nested x_ classes'
+    },
+    { id: 'gmail', predicate: "m.client = 'gmail'", weight: 2, description: 'the most common client' },
+    { id: 'apple-mail', predicate: "m.client IN ('apple-mail', 'apple-mail-mobile')", weight: 2, description: 'blockquote type=cite' },
+    { id: 'thunderbird', predicate: "m.client = 'thunderbird'", weight: 2, description: 'moz-cite-prefix' },
+    { id: 'unknown-client', predicate: "m.client = 'unknown'", weight: 2, description: 'no client fingerprint at all' },
+    { id: 'deep-quote', predicate: 'm.quote_depth_max >= 3', weight: 2, description: 'deeply nested threads' },
+    { id: 'forwarded', predicate: 'm.is_forward = 1', weight: 2, description: 'forwarded content must never be lost' },
+    { id: 'tables', predicate: 'm.table_count > 2', weight: 1, description: 'table heavy layout' },
+    { id: 'malformed-html', predicate: 'm.suspect_malformed_html = 1', weight: 1, description: 'markup that needs a forgiving parser' },
+    { id: 'legacy-charset', predicate: "m.charsets NOT LIKE '%utf-8%' AND m.charsets != ''", weight: 1, description: 'legacy charset decoding' },
+    { id: 'non-english', predicate: "m.lang NOT IN ('en', 'unknown')", weight: 2, description: 'multilingual attribution lines' },
+    { id: 'has-warnings', predicate: "m.warnings != ''", weight: 1, description: 'messages the parser complained about' },
+    { id: 'random-control', predicate: '1 = 1', weight: 4, description: 'unbiased slice for honest aggregate numbers' }
+];
+/**
+ * Draw a stratified sample.
+ *
+ * Strata overlap by design: a message can be an Outlook interleaved reply in Estonian. It is
+ * assigned to the first stratum that claims it, so the rarer strata listed earlier fill first, and
+ * no message appears twice.
+ */
+function stratifiedSample(store, options) {
+    const strata = options.strata ?? exports.DEFAULT_STRATA;
+    const exclude = options.exclude ?? new Set();
+    const totalWeight = strata.reduce((sum, stratum) => sum + stratum.weight, 0);
+    const base = ["f.status = 'ok'"];
+    if (options.traffic && options.traffic !== 'all') {
+        base.push(`m.traffic = '${options.traffic.replace(/'/g, '')}'`);
+    }
+    if (options.repliesOnly !== false) {
+        base.push('m.is_reply = 1');
+    }
+    const chosen = new Set();
+    const rows = [];
+    const coverage = [];
+    for (const stratum of strata) {
+        const quota = Math.max(1, Math.round((options.count * stratum.weight) / totalWeight));
+        const candidates = store
+            .query(`SELECT m.path AS path FROM messages m JOIN files f ON f.path = m.path WHERE ${[...base, stratum.predicate].join(' AND ')}`)
+            .map(row => row.path)
+            .filter(path => !exclude.has(path) && !chosen.has(path));
+        // Deterministic: hash each path with the seed and take the lowest values.
+        const ranked = candidates
+            .map(path => ({ path, score: (0, hash_1.deterministicUnit)(`${options.seed}:${stratum.id}`, path) }))
+            .sort((a, b) => a.score - b.score)
+            .slice(0, quota);
+        for (const item of ranked) {
+            chosen.add(item.path);
+            rows.push({ path: item.path, stratum: stratum.id });
+        }
+        coverage.push({
+            id: stratum.id,
+            requested: quota,
+            taken: ranked.length,
+            available: candidates.length,
+            description: stratum.description
+        });
+    }
+    return { rows, coverage };
+}
+//# sourceMappingURL=sample.js.map

package/dist/corpus/sample.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample.js","sourceRoot":"","sources":["../../src/corpus/sample.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;AA8EH,4CA+CC;AA3HD,uCAAiD;AAYjD;;;;;GAKG;AACU,QAAA,cAAc,GAAuB;IAC9C,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,+BAA+B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,2DAA2D,EAAE;IACtJ,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,0BAA0B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE;IAC9G,EAAE,EAAE,EAAE,kBAAkB,EAAE,SAAS,EAAE,oCAAoC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,yCAAyC,EAAE;IAC9I,EAAE,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,4BAA4B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,yBAAyB,EAAE;IAChH,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,2BAA2B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,+BAA+B,EAAE;IACpH,EAAE,EAAE,EAAE,0BAA0B,EAAE,SAAS,EAAE,+BAA+B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,uCAAuC,EAAE;IAC/I;QACI,EAAE,EAAE,SAAS;QACb,SAAS,EAAE,kEAAkE;QAC7E,MAAM,EAAE,CAAC;QACT,WAAW,EAAE,iDAAiD;KACjE;IACD,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE;IAClG,EAAE,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,iDAAiD,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE;IAClI,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,0BAA0B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACvG,EAAE,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,8BAA8B,EAAE;IACnH,EAAE,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,wBAAwB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,uBAAuB,EAAE;IAC1G,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,sCAAsC,EAAE;IAClH,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,oBAAoB,EAAE;IAC9F,EAAE,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE,8BAA8B,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,sCAAsC,EAAE;IACnI,EAAE,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE,oDAAoD,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,yBAAyB,EAAE;IAC5I,EAAE,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,iCAAiC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE;IAC7H,EAAE,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,sCAAsC,EAAE;IACrH,EAAE,EAAE,EAAE,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,6CAA6C,EAAE;CACtH,CAAC;AA0BF;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,KAAkB,EAAE,OAAsB;IACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,sBAAc,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,GAAG,EAAU,CAAC;IACrD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAE7E,MAAM,IAAI,GAAa,CAAC,iBAAiB,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAA6B,EAAE,CAAC;IAE9C,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;QACtF,MAAM,UAAU,GAAG,KAAK;aACnB,KAAK,CACF,+EAA+E,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAC9H;aACA,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;aACpB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7D,0EAA0E;QAC1E,MAAM,MAAM,GAAG,UAAU;aACpB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAA,wBAAiB,EAAC,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;aACxF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAErB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,SAAS,EAAE,KAAK;YAChB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,SAAS,EAAE,UAAU,CAAC,MAAM;YAC5B,WAAW,EAAE,OAAO,CAAC,WAAW;SACnC,CAAC,CAAC;IACP,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC9B,CAAC"}