@postalsys/email-content 0.0.1

package/dist/analyze.js

@@ -0,0 +1,274 @@
+"use strict";
+/**
+ * Content first API.
+ *
+ * `extract()` takes a raw RFC 5322 message and does its own MIME parsing. Most callers do not have
+ * one: they have already parsed the message, with mailparser, postal-mime, EmailEngine or an IMAP
+ * client, and hold `{ text, html, subject }`. Re-serialising that back into a message just to parse
+ * it again would be wasteful and lossy.
+ *
+ * `analyze()` is the same pipeline entered one stage later. Stage 1 is skipped, Stages 2 to 8 are
+ * identical, and the result has the same shape. Everything the extractor decides is derived from
+ * the bodies themselves, so nothing is lost by not having the envelope.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.senderTokensFrom = senderTokensFrom;
+exports.resolveHistory = resolveHistory;
+exports.analyze = analyze;
+const html_blocks_1 = require("./segment/html-blocks");
+const text_blocks_1 = require("./segment/text-blocks");
+const rules_1 = require("./classify/rules");
+const reconcile_1 = require("./classify/reconcile");
+const confidence_1 = require("./classify/confidence");
+const history_1 = require("./match/history");
+const types_1 = require("./types");
+const REPLY_SUBJECT = /^\s*(re|aw|sv|vs|vast|antw|res|odp|ответ)\s*(\[\d+\])?\s*:/i;
+function contentChars(blocks) {
+    return blocks.reduce((sum, block) => sum + block.content.length, 0);
+}
+function hasAnchor(blocks) {
+    return blocks.some(block => block.evidence.some(evidence => evidence.kind === 'reply-anchor' || evidence.kind === 'forward-anchor' || (evidence.kind === 'quote-container' && evidence.strength >= 0.85)));
+}
+/**
+ * Name tokens from a From header, for signature detection.
+ *
+ * Takes the display name words and the address local part, keeping tokens long enough to be
+ * distinctive. Short tokens are dropped because a two letter name fragment matches everywhere.
+ */
+function senderTokensFrom(from) {
+    if (!from) {
+        return [];
+    }
+    const display = from.replace(/<[^>]*>/g, ' ').replace(/["']/g, ' ');
+    const local = (from.match(/([\w.+-]{1,64})@/) || [])[1] ?? '';
+    const tokens = new Set();
+    for (const token of `${display} ${local.replace(/[._+-]+/g, ' ')}`.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
+        if (token.length >= 3) {
+            tokens.add(token);
+        }
+    }
+    return [...tokens];
+}
+function buildRepresentation(blocks, representation, config, history, senderTokens) {
+    const matches = history.length ? (0, history_1.matchBlocks)(blocks, history) : [];
+    return {
+        representation,
+        decisions: (0, rules_1.classifyBlocks)(blocks, { config, historyMatchedIndices: (0, history_1.matchedIndices)(matches), senderTokens }),
+        contentChars: contentChars(blocks),
+        hasAnchor: hasAnchor(blocks),
+        matches
+    };
+}
+/** Merge adjacent decisions carrying the same label, so the output is regions rather than lines. */
+function toSegments(decisions, config) {
+    const segments = [];
+    for (const decision of decisions) {
+        const block = decision.block;
+        const previous = segments[segments.length - 1];
+        const visibility = (0, confidence_1.visibilityFor)(decision.label, decision.confidence, config);
+        if (previous && previous.type === decision.label && previous.visibility === visibility && previous.source.mimePart === block.mimePart) {
+            previous.text = previous.text ? `${previous.text}\n${block.content}` : block.content;
+            if (block.html) {
+                previous.html = (previous.html ?? '') + block.html;
+            }
+            previous.source.end = block.end;
+            previous.confidence = Math.min(previous.confidence, decision.confidence);
+            for (const reason of decision.reasons) {
+                if (!previous.reasons.includes(reason)) {
+                    previous.reasons.push(reason);
+                }
+            }
+            continue;
+        }
+        if (block.features.isAllBlank && !previous) {
+            continue;
+        }
+        segments.push({
+            type: decision.label,
+            text: block.content,
+            ...(block.html ? { html: block.html } : {}),
+            confidence: decision.confidence,
+            visibility,
+            reasons: [...decision.reasons],
+            source: {
+                mimePart: block.mimePart,
+                representation: block.representation,
+                blockIndex: block.index,
+                start: block.start,
+                end: block.end,
+                ...(block.domPath ? { domPath: block.domPath } : {})
+            }
+        });
+    }
+    return segments.filter(segment => segment.text.trim().length > 0);
+}
+/** Turn whatever the caller supplied as history into searchable documents. */
+async function resolveHistory(options) {
+    if (options.historyDocuments?.length) {
+        return [...options.historyDocuments];
+    }
+    const documents = [];
+    for (let i = 0; i < (options.history?.length ?? 0); i++) {
+        const item = options.history[i];
+        try {
+            if (typeof item === 'string' || Buffer.isBuffer(item)) {
+                documents.push(await (0, history_1.buildHistoryDocument)(item, `history-${i}`, i === 0));
+            }
+            else if (item) {
+                documents.push((0, history_1.buildHistoryDocumentFromContent)(item.text ?? '', item.html ?? '', `history-${i}`, i === 0));
+            }
+        }
+        catch {
+            // An unusable ancestor simply contributes no evidence.
+        }
+    }
+    return documents;
+}
+/**
+ * Analyse an already parsed message body.
+ *
+ * Never throws. A body that cannot be segmented is returned whole as a single visible `UNKNOWN`
+ * segment, because showing too much is always preferable to showing nothing.
+ */
+async function analyze(content, options = {}) {
+    const config = options.confidence ?? confidence_1.DEFAULT_CONFIDENCE;
+    const warnings = [];
+    const text = typeof content.text === 'string' ? content.text : '';
+    const html = typeof content.html === 'string' ? content.html : '';
+    const isReply = Boolean(content.inReplyTo) || (content.references?.length ?? 0) > 0 || REPLY_SUBJECT.test(content.subject ?? '');
+    const base = {
+        originalAvailable: true,
+        extractorVersion: types_1.EXTRACTOR_VERSION,
+        isReply
+    };
+    if (!text.trim() && !html.trim()) {
+        return {
+            ...base,
+            segments: [],
+            hiddenSegments: [],
+            overallConfidence: 1,
+            warnings: [{ code: 'no_body_parts' }],
+            primaryRepresentation: 'none',
+            historyUsed: false
+        };
+    }
+    const senderTokens = senderTokensFrom(content.from);
+    let history = [];
+    try {
+        history = await resolveHistory(options);
+    }
+    catch {
+        // History is optional evidence. Failing to build it is never fatal.
+    }
+    try {
+        let textResult = null;
+        let htmlResult = null;
+        let primaryBlocks = [];
+        if (text.trim()) {
+            const segmented = (0, text_blocks_1.segmentTextBlocks)(text, content.textPartId ?? 'TEXT');
+            textResult = buildRepresentation(segmented.blocks, 'text', config, history, senderTokens);
+            primaryBlocks = segmented.blocks;
+        }
+        if (html.trim()) {
+            const segmented = (0, html_blocks_1.segmentHtmlBlocks)(html, content.htmlPartId ?? 'HTML');
+            if (segmented.degraded) {
+                warnings.push({ code: 'html_parse_error' });
+            }
+            htmlResult = buildRepresentation(segmented.blocks, 'html', config, history, senderTokens);
+            if (!textResult) {
+                primaryBlocks = segmented.blocks;
+            }
+        }
+        if (!textResult && !htmlResult) {
+            return {
+                ...base,
+                segments: [],
+                hiddenSegments: [],
+                overallConfidence: 1,
+                warnings: [...warnings, { code: 'no_body_parts' }],
+                primaryRepresentation: 'none',
+                historyUsed: false
+            };
+        }
+        const outcome = (0, reconcile_1.pickPrimary)(textResult, htmlResult);
+        if (outcome.primary === htmlResult && htmlResult) {
+            primaryBlocks = htmlResult.decisions.map(decision => decision.block);
+        }
+        else if (textResult) {
+            primaryBlocks = textResult.decisions.map(decision => decision.block);
+        }
+        const decisions = (0, reconcile_1.reconcile)(outcome);
+        const segments = toSegments(decisions, config);
+        const visible = segments.filter(segment => segment.visibility !== 'hide');
+        const hidden = segments.filter(segment => segment.visibility === 'hide');
+        // Invariant: a message with body text always yields something visible.
+        const bodyHasText = decisions.some(decision => decision.block.content.trim().length > 0);
+        if (bodyHasText && !visible.length) {
+            warnings.push({ code: 'extraction_degraded', detail: 'no visible segment survived classification' });
+            for (const segment of segments) {
+                segment.visibility = 'show';
+                segment.type = 'UNKNOWN';
+                segment.reasons.push('degraded_to_unknown');
+            }
+            return {
+                ...base,
+                segments,
+                hiddenSegments: [],
+                overallConfidence: 0.5,
+                warnings,
+                primaryRepresentation: outcome.primary.representation,
+                historyUsed: history.length > 0,
+                ...(options.explain ? { diagnostics: buildDiagnostics(primaryBlocks, decisions) } : {})
+            };
+        }
+        return {
+            ...base,
+            segments: visible,
+            hiddenSegments: hidden,
+            overallConfidence: (0, confidence_1.overallConfidence)(segments),
+            warnings,
+            primaryRepresentation: outcome.primary.representation,
+            historyUsed: history.length > 0,
+            ...(options.explain ? { diagnostics: buildDiagnostics(primaryBlocks, decisions) } : {})
+        };
+    }
+    catch (err) {
+        const fallback = text || html;
+        return {
+            ...base,
+            segments: fallback.trim()
+                ? [
+                    {
+                        type: 'UNKNOWN',
+                        text: fallback,
+                        confidence: 0.5,
+                        visibility: 'show',
+                        reasons: ['degraded_to_unknown'],
+                        source: { mimePart: 'TEXT', representation: 'text', blockIndex: 0, start: 0, end: fallback.length }
+                    }
+                ]
+                : [],
+            hiddenSegments: [],
+            overallConfidence: 0.5,
+            warnings: [...warnings, { code: 'extraction_degraded', detail: err instanceof Error ? err.message.slice(0, 200) : 'unknown' }],
+            primaryRepresentation: 'none',
+            historyUsed: history.length > 0
+        };
+    }
+}
+function buildDiagnostics(blocks, decisions) {
+    return {
+        primaryPart: blocks[0]?.mimePart ?? null,
+        blocks: decisions.map(decision => ({
+            index: decision.block.index,
+            kind: decision.block.kind,
+            label: decision.label,
+            confidence: decision.confidence,
+            reasons: decision.reasons,
+            evidence: decision.block.evidence.map(item => ({ id: item.id, kind: item.kind, strength: item.strength })),
+            features: decision.block.features,
+            preview: decision.block.content.slice(0, 120)
+        }))
+    };
+}
+//# sourceMappingURL=analyze.js.map

package/dist/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;AA0FH,4CAaC;AAqED,wCAkBC;AAQD,0BAuIC;AA3UD,uDAA0D;AAC1D,uDAA0D;AAE1D,4CAAkD;AAClD,oDAAyF;AACzF,sDAAoH;AACpH,6CAA2I;AAC3I,mCAAqF;AA4DrF,MAAM,aAAa,GAAG,6DAA6D,CAAC;AAEpF,SAAS,YAAY,CAAC,MAAwB;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB;IACvC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CACvB,KAAK,CAAC,QAAQ,CAAC,IAAI,CACf,QAAQ,CAAC,EAAE,CACP,QAAQ,CAAC,IAAI,KAAK,cAAc,IAAI,QAAQ,CAAC,IAAI,KAAK,gBAAgB,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,iBAAiB,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,CACnJ,CACJ,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,SAAgB,gBAAgB,CAAC,IAA+B;IAC5D,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,GAAG,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACxG,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,mBAAmB,CACxB,MAAe,EACf,cAA+B,EAC/B,MAAwB,EACxB,OAAmC,EACnC,YAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAA,qBAAW,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO;QACH,cAAc;QACd,SAAS,EAAE,IAAA,sBAAc,EAAC,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,EAAE,IAAA,wBAAc,EAAC,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;QAC3G,YAAY,EAAE,YAAY,CAAC,MAAM,CAAC;QAClC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC;QAC5B,OAAO;KACV,CAAC;AACN,CAAC;AAED,oGAAoG;AACpG,SAAS,UAAU,CAAC,SAA4C,EAAE,MAAwB;IACtF,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAA,0BAAa,EAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE9E,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,KAAK,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpI,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;YACrF,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;YACvD,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YAChC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;YACzE,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACpC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACrC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;YACD,SAAS;QACb,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,SAAS;QACb,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,QAAQ,CAAC,KAAK;YACpB,IAAI,EAAE,KAAK,CAAC,OAAO;YACnB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,UAAU;YACV,OAAO,EAAE,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC9B,MAAM,EAAE;gBACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvD;SACJ,CAAC,CAAC;IACP,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,8EAA8E;AACvE,KAAK,UAAU,cAAc,CAAC,OAAuB;IACxD,IAAI,OAAO,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,IAAI,GAAI,OAAO,CAAC,OAAyD,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpD,SAAS,CAAC,IAAI,CAAC,MAAM,IAAA,8BAAoB,EAAC,IAAI,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9E,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,SAAS,CAAC,IAAI,CAAC,IAAA,yCAA+B,EAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/G,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,uDAAuD;QAC3D,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,OAAO,CAAC,OAAuB,EAAE,UAA0B,EAAE;IAC/E,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,IAAI,+BAAkB,CAAC;IACxD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,MAAM,IAAI,GAAG,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,MAAM,IAAI,GAAG,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAEjI,MAAM,IAAI,GAAG;QACT,iBAAiB,EAAE,IAAa;QAChC,gBAAgB,EAAE,yBAAiB;QACnC,OAAO;KACV,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,OAAO;YACH,GAAG,IAAI;YACP,QAAQ,EAAE,EAAE;YACZ,cAAc,EAAE,EAAE;YAClB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;YACrC,qBAAqB,EAAE,MAAM;YAC7B,WAAW,EAAE,KAAK;SACrB,CAAC;IACN,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,IAAI,OAAO,GAAsB,EAAE,CAAC;IACpC,IAAI,CAAC;QACD,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACL,oEAAoE;IACxE,CAAC;IAED,IAAI,CAAC;QACD,IAAI,UAAU,GAAgC,IAAI,CAAC;QACnD,IAAI,UAAU,GAAgC,IAAI,CAAC;QACnD,IAAI,aAAa,GAAY,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACd,MAAM,SAAS,GAAG,IAAA,+BAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,CAAC;YACxE,UAAU,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;YAC1F,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACd,MAAM,SAAS,GAAG,IAAA,+BAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,CAAC;YACxE,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAChD,CAAC;YACD,UAAU,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;YAC1F,IAAI,CAAC,UAAU,EAAE,CAAC;gBACd,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC;YACrC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO;gBACH,GAAG,IAAI;gBACP,QAAQ,EAAE,EAAE;gBACZ,cAAc,EAAE,EAAE;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;gBAClD,qBAAqB,EAAE,MAAM;gBAC7B,WAAW,EAAE,KAAK;aACrB,CAAC;QACN,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,uBAAW,EAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,OAAO,KAAK,UAAU,IAAI,UAAU,EAAE,CAAC;YAC/C,aAAa,GAAG,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzE,CAAC;aAAM,IAAI,UAAU,EAAE,CAAC;YACpB,aAAa,GAAG,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,SAAS,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;QAEzE,uEAAuE;QACvE,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzF,IAAI,WAAW,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,4CAA4C,EAAE,CAAC,CAAC;YACrG,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,OAAO,CAAC,UAAU,GAAG,MAAM,CAAC;gBAC5B,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;gBACzB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YAChD,CAAC;YACD,OAAO;gBACH,GAAG,IAAI;gBACP,QAAQ;gBACR,cAAc,EAAE,EAAE;gBAClB,iBAAiB,EAAE,GAAG;gBACtB,QAAQ;gBACR,qBAAqB,EAAE,OAAO,CAAC,OAAO,CAAC,cAAc;gBACrD,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;gBAC/B,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,gBAAgB,CAAC,aAAa,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1F,CAAC;QACN,CAAC;QAED,OAAO;YACH,GAAG,IAAI;YACP,QAAQ,EAAE,OAAO;YACjB,cAAc,EAAE,MAAM;YACtB,iBAAiB,EAAE,IAAA,8BAAiB,EAAC,QAAQ,CAAC;YAC9C,QAAQ;YACR,qBAAqB,EAAE,OAAO,CAAC,OAAO,CAAC,cAAc;YACrD,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC/B,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,gBAAgB,CAAC,aAAa,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1F,CAAC;IACN,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,IAAI,IAAI,IAAI,CAAC;QAC9B,OAAO;YACH,GAAG,IAAI;YACP,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;gBACrB,CAAC,CAAC;oBACI;wBACI,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE,GAAG;wBACf,UAAU,EAAE,MAAM;wBAClB,OAAO,EAAE,CAAC,qBAAqB,CAAC;wBAChC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE;qBACtG;iBACJ;gBACH,CAAC,CAAC,EAAE;YACR,cAAc,EAAE,EAAE;YAClB,iBAAiB,EAAE,GAAG;YACtB,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YAC9H,qBAAqB,EAAE,MAAM;YAC7B,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;SAClC,CAAC;IACN,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAwB,EAAE,SAA4C;IAC5F,OAAO;QACH,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI;QACxC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;YAC3B,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;YACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1G,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAA8C;YACvE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAChD,CAAC,CAAC;KACN,CAAC;AACN,CAAC"}

package/dist/annotate/client.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Minimal OpenAI chat completions client.
+ *
+ * No SDK dependency: this project ships three runtime dependencies and none of them opens a socket.
+ * Adding a vendor SDK for one optional command would change that materially, and the surface used
+ * here is a single POST with a JSON schema response format.
+ *
+ * This is the one place in the codebase that sends message content off the machine. It is only
+ * reachable from `email-content annotate`, never from `extract()` or the corpus scanner. See
+ * docs/threat-model.md, "Configured exception: LLM annotation".
+ */
+export interface OpenAIConfig {
+    apiKey: string;
+    baseUrl: string;
+    model: string;
+    /** Per request timeout in milliseconds. */
+    timeoutMs: number;
+    maxRetries: number;
+}
+export interface Usage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    requests: number;
+}
+export interface CompletionRequest {
+    system: string;
+    user: string;
+    schemaName: string;
+    schema: Record<string, unknown>;
+}
+/**
+ * The narrow surface the annotator depends on.
+ *
+ * Depending on this rather than on the concrete client is what lets the annotator's validation
+ * logic be tested without a network call, which matters because that validation is the guard
+ * against a model returning indices that do not exist.
+ */
+export interface CompletionClient {
+    readonly model: string;
+    complete<T>(request: CompletionRequest): Promise<T>;
+}
+export declare class OpenAIError extends Error {
+    readonly status: number;
+    readonly retryable: boolean;
+    constructor(message: string, status: number, retryable: boolean);
+}
+/** Load configuration from the environment, after process.loadEnvFile(). */
+export declare function configFromEnv(model: string, overrides?: Partial<OpenAIConfig>): OpenAIConfig;
+export declare class OpenAIClient implements CompletionClient {
+    private readonly config;
+    readonly usage: Usage;
+    constructor(config: OpenAIConfig);
+    get model(): string;
+    /**
+     * Request a structured completion and return the parsed object.
+     *
+     * Retries on rate limits and transient server errors with exponential backoff plus jitter. A
+     * schema violation is not retried: strict mode means the model already failed to satisfy the
+     * contract, and a retry would usually fail the same way.
+     */
+    complete<T>(request: CompletionRequest): Promise<T>;
+}
+//# sourceMappingURL=client.d.ts.map

package/dist/annotate/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/annotate/client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,KAAK;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACvD;AAED,qBAAa,WAAY,SAAQ,KAAK;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;gBAEhB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO;CAMlE;AAED,4EAA4E;AAC5E,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,YAAY,CAahG;AAMD,qBAAa,YAAa,YAAW,gBAAgB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAyE;gBAElF,MAAM,EAAE,YAAY;IAIhC,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;OAMG;IACG,QAAQ,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC;CAoF5D"}

package/dist/annotate/client.js

@@ -0,0 +1,139 @@
+"use strict";
+/**
+ * Minimal OpenAI chat completions client.
+ *
+ * No SDK dependency: this project ships three runtime dependencies and none of them opens a socket.
+ * Adding a vendor SDK for one optional command would change that materially, and the surface used
+ * here is a single POST with a JSON schema response format.
+ *
+ * This is the one place in the codebase that sends message content off the machine. It is only
+ * reachable from `email-content annotate`, never from `extract()` or the corpus scanner. See
+ * docs/threat-model.md, "Configured exception: LLM annotation".
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIClient = exports.OpenAIError = void 0;
+exports.configFromEnv = configFromEnv;
+class OpenAIError extends Error {
+    status;
+    retryable;
+    constructor(message, status, retryable) {
+        super(message);
+        this.name = 'OpenAIError';
+        this.status = status;
+        this.retryable = retryable;
+    }
+}
+exports.OpenAIError = OpenAIError;
+/** Load configuration from the environment, after process.loadEnvFile(). */
+function configFromEnv(model, overrides = {}) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error('OPENAI_API_KEY is not set. Copy .env.example to .env and fill it in.');
+    }
+    return {
+        apiKey,
+        baseUrl: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1',
+        model,
+        timeoutMs: 120000,
+        maxRetries: 5,
+        ...overrides
+    };
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+class OpenAIClient {
+    config;
+    usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0, requests: 0 };
+    constructor(config) {
+        this.config = config;
+    }
+    get model() {
+        return this.config.model;
+    }
+    /**
+     * Request a structured completion and return the parsed object.
+     *
+     * Retries on rate limits and transient server errors with exponential backoff plus jitter. A
+     * schema violation is not retried: strict mode means the model already failed to satisfy the
+     * contract, and a retry would usually fail the same way.
+     */
+    async complete(request) {
+        let lastError = null;
+        for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
+            if (attempt > 0) {
+                // Full jitter, capped, so a burst of workers does not retry in lockstep.
+                const backoff = Math.min(30000, 500 * 2 ** (attempt - 1));
+                await sleep(Math.random() * backoff);
+            }
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
+            try {
+                const response = await fetch(`${this.config.baseUrl}/chat/completions`, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        Authorization: `Bearer ${this.config.apiKey}`,
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                        model: this.config.model,
+                        messages: [
+                            { role: 'system', content: request.system },
+                            { role: 'user', content: request.user }
+                        ],
+                        response_format: {
+                            type: 'json_schema',
+                            json_schema: { name: request.schemaName, strict: true, schema: request.schema }
+                        }
+                    })
+                });
+                if (!response.ok) {
+                    const retryable = response.status === 429 || response.status >= 500;
+                    let detail = `HTTP ${response.status}`;
+                    try {
+                        const body = (await response.json());
+                        if (body.error?.message) {
+                            detail = body.error.message.slice(0, 300);
+                        }
+                    }
+                    catch {
+                        // Body was not JSON. The status alone is enough.
+                    }
+                    const error = new OpenAIError(detail, response.status, retryable);
+                    if (!retryable) {
+                        throw error;
+                    }
+                    lastError = error;
+                    continue;
+                }
+                const body = (await response.json());
+                this.usage.requests++;
+                this.usage.promptTokens += body.usage?.prompt_tokens ?? 0;
+                this.usage.completionTokens += body.usage?.completion_tokens ?? 0;
+                this.usage.totalTokens += body.usage?.total_tokens ?? 0;
+                const choice = body.choices?.[0];
+                if (choice?.finish_reason === 'length') {
+                    throw new OpenAIError('response truncated by token limit', 0, false);
+                }
+                const content = choice?.message?.content;
+                if (!content) {
+                    throw new OpenAIError('empty completion', 0, false);
+                }
+                return JSON.parse(content);
+            }
+            catch (err) {
+                if (err instanceof OpenAIError && !err.retryable) {
+                    throw err;
+                }
+                lastError = err instanceof Error ? err : new Error(String(err));
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }
+        throw lastError ?? new Error('request failed');
+    }
+}
+exports.OpenAIClient = OpenAIClient;
+//# sourceMappingURL=client.js.map

package/dist/annotate/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/annotate/client.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAkDH,sCAaC;AA1BD,MAAa,WAAY,SAAQ,KAAK;IACzB,MAAM,CAAS;IACf,SAAS,CAAU;IAE5B,YAAY,OAAe,EAAE,MAAc,EAAE,SAAkB;QAC3D,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;CACJ;AAVD,kCAUC;AAED,4EAA4E;AAC5E,SAAgB,aAAa,CAAC,KAAa,EAAE,YAAmC,EAAE;IAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO;QACH,MAAM;QACN,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,2BAA2B;QACnE,KAAK;QACL,SAAS,EAAE,MAAM;QACjB,UAAU,EAAE,CAAC;QACb,GAAG,SAAS;KACf,CAAC;AACN,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAa,YAAY;IACJ,MAAM,CAAe;IAC7B,KAAK,GAAU,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAE9F,YAAY,MAAoB;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAI,OAA0B;QACxC,IAAI,SAAS,GAAiB,IAAI,CAAC;QAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACjE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBACd,yEAAyE;gBACzE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;YACzC,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE1E,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,mBAAmB,EAAE;oBACpE,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACL,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;wBAC7C,cAAc,EAAE,kBAAkB;qBACrC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACjB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;wBACxB,QAAQ,EAAE;4BACN,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE;4BAC3C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;yBAC1C;wBACD,eAAe,EAAE;4BACb,IAAI,EAAE,aAAa;4BACnB,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;yBAClF;qBACJ,CAAC;iBACL,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACf,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC;oBACpE,IAAI,MAAM,GAAG,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACvC,IAAI,CAAC;wBACD,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqC,CAAC;wBACzE,IAAI,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;4BACtB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAC9C,CAAC;oBACL,CAAC;oBAAC,MAAM,CAAC;wBACL,iDAAiD;oBACrD,CAAC;oBACD,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;oBAClE,IAAI,CAAC,SAAS,EAAE,CAAC;wBACb,MAAM,KAAK,CAAC;oBAChB,CAAC;oBACD,SAAS,GAAG,KAAK,CAAC;oBAClB,SAAS;gBACb,CAAC;gBAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAGlC,CAAC;gBAEF,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACtB,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC;gBAC1D,IAAI,CAAC,KAAK,CAAC,gBAAgB,IAAI,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,CAAC,CAAC;gBAClE,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;gBAExD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;gBACjC,IAAI,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,IAAI,WAAW,CAAC,mCAAmC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;gBACzE,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;gBACzC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACX,MAAM,IAAI,WAAW,CAAC,kBAAkB,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAC;YACpC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,GAAG,YAAY,WAAW,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;oBAC/C,MAAM,GAAG,CAAC;gBACd,CAAC;gBACD,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;oBAAS,CAAC;gBACP,YAAY,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACnD,CAAC;CACJ;AAvGD,oCAuGC"}

package/dist/annotate/llm.d.ts

@@ -0,0 +1,46 @@
+/**
+ * LLM block annotator.
+ *
+ * Produces labels for an already segmented message. It does not replace the segmenter: the model
+ * only ever sees blocks the deterministic pipeline produced, and only ever returns labels for those
+ * block indices. That keeps the annotation aligned with the pipeline it is meant to evaluate, and
+ * keeps generated text out of the system entirely.
+ *
+ * These labels are weak supervision, not ground truth. They are stored with their model and prompt
+ * version, kept separate from human labels, and are meant for bootstrapping, disagreement mining
+ * and sampling, exactly as the project's plan requires of any non human label source.
+ */
+import { type Block } from '../segment/types';
+import { type Label } from '../types';
+import { type CompletionClient } from './client';
+/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
+export declare const MAX_BLOCK_CHARS = 700;
+/** Blocks per message. Longer messages are truncated with a recorded flag. */
+export declare const MAX_BLOCKS = 60;
+export interface LlmLabel {
+    index: number;
+    label: Label;
+    confidence: number;
+}
+export interface AnnotationResult {
+    labels: LlmLabel[];
+    model: string;
+    promptVersion: string;
+    /** Blocks that were sent but came back unlabelled, defaulted to UNKNOWN. */
+    missing: number[];
+    /** Indices the model invented that do not exist. Discarded. */
+    spurious: number[];
+    blocksSent: number;
+    blocksTotal: number;
+}
+/** Blocks worth spending a token on. Blank blocks carry no decision. */
+export declare function annotatableBlocks(blocks: readonly Block[]): Block[];
+/**
+ * Annotate one message's blocks.
+ *
+ * @param client configured OpenAI client
+ * @param blocks blocks from the primary representation
+ * @param isReply whether the message carries threading headers
+ */
+export declare function annotateBlocks(client: CompletionClient, blocks: readonly Block[], isReply: boolean): Promise<AnnotationResult>;
+//# sourceMappingURL=llm.d.ts.map

package/dist/annotate/llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAGjD,2FAA2F;AAC3F,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,8EAA8E;AAC9E,eAAO,MAAM,UAAU,KAAK,CAAC;AAE7B,MAAM,WAAW,QAAQ;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,wEAAwE;AACxE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,KAAK,EAAE,CAEnE;AAkBD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAgEpI"}

package/dist/annotate/llm.js

@@ -0,0 +1,107 @@
+"use strict";
+/**
+ * LLM block annotator.
+ *
+ * Produces labels for an already segmented message. It does not replace the segmenter: the model
+ * only ever sees blocks the deterministic pipeline produced, and only ever returns labels for those
+ * block indices. That keeps the annotation aligned with the pipeline it is meant to evaluate, and
+ * keeps generated text out of the system entirely.
+ *
+ * These labels are weak supervision, not ground truth. They are stored with their model and prompt
+ * version, kept separate from human labels, and are meant for bootstrapping, disagreement mining
+ * and sampling, exactly as the project's plan requires of any non human label source.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_BLOCKS = exports.MAX_BLOCK_CHARS = void 0;
+exports.annotatableBlocks = annotatableBlocks;
+exports.annotateBlocks = annotateBlocks;
+const prompt_1 = require("./prompt");
+/** Characters of each block sent to the model. Enough for a decision, bounded for cost. */
+exports.MAX_BLOCK_CHARS = 700;
+/** Blocks per message. Longer messages are truncated with a recorded flag. */
+exports.MAX_BLOCKS = 60;
+/** Blocks worth spending a token on. Blank blocks carry no decision. */
+function annotatableBlocks(blocks) {
+    return blocks.filter(block => block.content.trim().length > 0);
+}
+function toPromptBlocks(blocks) {
+    return blocks.slice(0, exports.MAX_BLOCKS).map(block => {
+        const text = block.content.trim();
+        const truncated = text.length > exports.MAX_BLOCK_CHARS;
+        return {
+            index: block.index,
+            text: truncated ? `${text.slice(0, exports.MAX_BLOCK_CHARS)}...` : text,
+            truncated,
+            // Both representations reduce to one nesting number: ">" depth in plain text,
+            // blockquote nesting in HTML. Without it the annotator cannot tell a quoted greeting
+            // from a freshly written one.
+            quoteDepth: Math.max(block.features.quoteDepth, block.features.blockquoteDepth)
+        };
+    });
+}
+/**
+ * Annotate one message's blocks.
+ *
+ * @param client configured OpenAI client
+ * @param blocks blocks from the primary representation
+ * @param isReply whether the message carries threading headers
+ */
+async function annotateBlocks(client, blocks, isReply) {
+    const candidates = annotatableBlocks(blocks);
+    const sent = toPromptBlocks(candidates);
+    if (!sent.length) {
+        return {
+            labels: [],
+            model: client.model,
+            promptVersion: prompt_1.PROMPT_VERSION,
+            missing: [],
+            spurious: [],
+            blocksSent: 0,
+            blocksTotal: candidates.length
+        };
+    }
+    const response = await client.complete({
+        system: prompt_1.SYSTEM_PROMPT,
+        user: (0, prompt_1.renderUserPrompt)(sent, isReply),
+        schemaName: 'block_labels',
+        schema: prompt_1.RESPONSE_SCHEMA
+    });
+    // Validate against the blocks we actually sent. The schema constrains shape and label values,
+    // but nothing stops the model returning an index we never asked about, or omitting one.
+    const expected = new Set(sent.map(block => block.index));
+    const seen = new Set();
+    const labels = [];
+    const spurious = [];
+    for (const item of response.labels ?? []) {
+        if (!expected.has(item.index) || seen.has(item.index)) {
+            spurious.push(item.index);
+            continue;
+        }
+        seen.add(item.index);
+        labels.push({
+            index: item.index,
+            label: item.label,
+            confidence: Number.isFinite(item.confidence) ? Math.min(1, Math.max(0, item.confidence)) : 0.5
+        });
+    }
+    // A block the model declined to label is UNKNOWN, which stays visible. Silently dropping it
+    // would let an omission look like agreement later.
+    const missing = [];
+    for (const index of expected) {
+        if (!seen.has(index)) {
+            missing.push(index);
+            labels.push({ index, label: 'UNKNOWN', confidence: 0 });
+        }
+    }
+    labels.sort((a, b) => a.index - b.index);
+    return {
+        labels,
+        model: client.model,
+        promptVersion: prompt_1.PROMPT_VERSION,
+        missing,
+        spurious,
+        blocksSent: sent.length,
+        blocksTotal: candidates.length
+    };
+}
+//# sourceMappingURL=llm.js.map