@postalsys/email-content 0.0.1

package/dist/annotate/llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/annotate/llm.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;;AAgCH,8CAEC;AAyBD,wCAgEC;AAtHD,qCAA8G;AAE9G,2FAA2F;AAC9E,QAAA,eAAe,GAAG,GAAG,CAAC;AAEnC,8EAA8E;AACjE,QAAA,UAAU,GAAG,EAAE,CAAC;AAoB7B,wEAAwE;AACxE,SAAgB,iBAAiB,CAAC,MAAwB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,MAAwB;IAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAU,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,uBAAe,CAAC;QAChD,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAe,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAC/D,SAAS;YACT,8EAA8E;YAC9E,qFAAqF;YACrF,8BAA8B;YAC9B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC;SAClF,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAAC,MAAwB,EAAE,MAAwB,EAAE,OAAgB;IACrG,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAExC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO;YACH,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,aAAa,EAAE,uBAAc;YAC7B,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,UAAU,CAAC,MAAM;SACjC,CAAC;IACN,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAoE;QACtG,MAAM,EAAE,sBAAa;QACrB,IAAI,EAAE,IAAA,yBAAgB,EAAC,IAAI,EAAE,OAAO,CAAC;QACrC,UAAU,EAAE,cAAc;QAC1B,MAAM,EAAE,wBAAe;KAC1B,CAAC,CAAC;IAEH,8FAA8F;IAC9F,wFAAwF;IACxF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC;YACR,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;SACjG,CAAC,CAAC;IACP,CAAC;IAED,4FAA4F;IAC5F,mDAAmD;IACnD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzC,OAAO;QACH,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,uBAAc;QAC7B,OAAO;QACP,QAAQ;QACR,UAAU,EAAE,IAAI,CAAC,MAAM;QACvB,WAAW,EAAE,UAAU,CAAC,MAAM;KACjC,CAAC;AACN,CAAC"}

package/dist/annotate/prompt.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The annotation prompt and its response schema.
+ *
+ * Versioned, because a prompt change invalidates comparability between annotation runs exactly the
+ * way a code change does. The version is stored with every label.
+ *
+ * Two properties are enforced by construction rather than by asking nicely:
+ *
+ *   1. The model receives numbered blocks and returns labels for those numbers. It is never asked
+ *      to produce text, so it cannot rewrite, summarise or paraphrase the sender's words. The
+ *      "select, never generate" rule holds even with a generative model in the loop.
+ *   2. The response schema is strict, with a closed label enum, so a malformed or creative answer
+ *      is rejected by the API rather than parsed hopefully at this end.
+ */
+export declare const PROMPT_VERSION = "annotate-v3";
+export declare const SYSTEM_PROMPT = "You label blocks of an email message so a chat application can decide what to show.\n\nYou are given the message split into numbered blocks, in order. Return exactly one label for every\nblock index you are given. Never return text, only indices and labels.\n\nLABELS\n\nNEW_CONTENT\n  Text the sender wrote themselves in this message. This is the default when unsure between\n  NEW_CONTENT and anything else.\n\nINLINE_QUOTE\n  Old text the sender quoted deliberately in order to answer it here, point by point. The giveaway\n  is that the sender's own new text appears after it and responds to it. Inline quotes must stay\n  visible, because the answer beside them is meaningless without them.\n\nREPLY_HISTORY\n  A copy of an earlier message that the sender's mail client attached automatically. Usually the\n  whole previous message, sitting at the end, introduced by an attribution line such as\n  \"On <date>, <person> wrote:\", or \"-----Original Message-----\", or an embedded\n  From:/Sent:/To:/Subject: block. This is the only label whose content gets hidden.\n\nFORWARDED_CONTENT\n  A message the sender forwarded on purpose, usually after \"Begin forwarded message:\" or\n  \"---------- Forwarded message ----------\". It is content the sender meant to pass on, so it is\n  kept and collapsed, never hidden.\n\nSIGNATURE\n  The sender's own sign-off block: name, role, company, phone, links, \"Sent from my phone\".\n\nDISCLAIMER\n  Boilerplate attached by a system, gateway or template independently of what the sender wrote:\n  confidentiality and liability notices, virus scanning notices, company registration and VAT\n  numbers, registered office addresses, unsubscribe and preference footers, \"powered by\" branding,\n  and standing promotional footers that repeat on every notification from a service.\n\n  Guard: if the boilerplate is the whole message, label it NEW_CONTENT. A newsletter is not a\n  disclaimer with nothing attached to it. When there is no other content, the boilerplate is the\n  content.\n\nUNKNOWN\n  Use when you genuinely cannot tell. Prefer this over guessing REPLY_HISTORY.\n\nHOW TO DECIDE\n\nThe important distinction is INLINE_QUOTE versus REPLY_HISTORY. Both are old text. The difference\nis purpose:\n\n  - If quoted text is followed by the sender's answer to it, and especially if this alternates\n    several times, it is INLINE_QUOTE.\n  - If quoted text runs to the end of the message as one uninterrupted copy of the previous mail,\n    it is REPLY_HISTORY.\n\nQuote characters such as \">\" are a hint, not proof. Some clients embed the previous message with no\nquote markers at all, as plain paragraphs after an attribution line: that is still REPLY_HISTORY.\nSome senders paste text that merely looks like an email; that is NEW_CONTENT.\n\nQUOTE DEPTH\n\nEach block is marked with its quote depth, exactly as the message shows it. Depth 0 means the block\nwas not quoted. Depth 1 means it was quoted once, depth 2 means it was quoted inside a quote, and so\non. Depth is structural evidence, not a verdict.\n\nLong threads nest: depth 1 is the message being replied to, depth 2 is the one before that. In a\nnested thread the sender's own new text is almost always at depth 0, and everything at depth 1 or\ndeeper is history, including greetings, thanks, questions and sign-offs. Text reading like a fresh\npersonal message is still REPLY_HISTORY if it is quoted, because someone wrote it in an earlier\nmessage, not in this one.\n\nREGION WINS OVER CONTENT TYPE\n\nA message is made of regions: what the sender wrote now, history their client attached, material\nthey forwarded on purpose. Signatures, greetings and legal footers exist inside all of them.\n\nLabel every block by the region it belongs to, never by what the text looks like. The question is\nalways \"which message does this block belong to?\".\n\n  - A signature belonging to this sender, in this message, is SIGNATURE.\n  - A signature inside quoted history is REPLY_HISTORY. It was someone else's sign-off, written in\n    an earlier message.\n  - A legal disclaimer inside a forwarded message is FORWARDED_CONTENT.\n  - A \"--\" delimiter inside a quote is part of that quote.\n\nThis applies to every footer type, not just signatures. A confidentiality notice sitting inside a\nforward is FORWARDED_CONTENT, not DISCLAIMER.\n\nCOST OF ERRORS\n\nMarking the sender's own words as REPLY_HISTORY hides them from the reader, who may never find out\nwhat was said. Leaving old text visible is only untidy. Treat the first mistake as ten times worse\nthan the second. When you are torn between NEW_CONTENT and REPLY_HISTORY, choose NEW_CONTENT. When\nyou are torn between INLINE_QUOTE and REPLY_HISTORY, choose INLINE_QUOTE.\n\nCONFIDENCE\n\nGive each label a confidence between 0 and 1. Use below 0.7 whenever the block is ambiguous, short,\nor could plausibly take another label.";
+export declare const RESPONSE_SCHEMA: Record<string, unknown>;
+export interface PromptBlock {
+    index: number;
+    text: string;
+    truncated: boolean;
+    /** Quote nesting depth, as the message shows it. Zero means not quoted. */
+    quoteDepth: number;
+}
+/**
+ * Render the user message.
+ *
+ * Deliberately minimal context: block text, quote depth, and whether the message is a reply. No
+ * subject, no addresses, no headers. Those would help a little and would send noticeably more
+ * personal data to a third party, which is the wrong trade for a marginal accuracy gain.
+ *
+ * Quote depth is shown because a human annotator reading the raw message would see the ">" markers
+ * and the visual nesting. Hiding it made the first version of this prompt label quoted greetings
+ * inside a five deep Estonian thread as freshly written text, which is exactly the mistake the
+ * depth marker prevents.
+ */
+export declare function renderUserPrompt(blocks: readonly PromptBlock[], isReply: boolean): string;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/annotate/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../src/annotate/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,eAAO,MAAM,cAAc,gBAAgB,CAAC;AAE5C,eAAO,MAAM,aAAa,gzJAgGa,CAAC;AAExC,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAsBnD,CAAC;AAEF,MAAM,WAAW,WAAW;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,MAAM,CAYzF"}

package/dist/annotate/prompt.js

@@ -0,0 +1,165 @@
+"use strict";
+/**
+ * The annotation prompt and its response schema.
+ *
+ * Versioned, because a prompt change invalidates comparability between annotation runs exactly the
+ * way a code change does. The version is stored with every label.
+ *
+ * Two properties are enforced by construction rather than by asking nicely:
+ *
+ *   1. The model receives numbered blocks and returns labels for those numbers. It is never asked
+ *      to produce text, so it cannot rewrite, summarise or paraphrase the sender's words. The
+ *      "select, never generate" rule holds even with a generative model in the loop.
+ *   2. The response schema is strict, with a closed label enum, so a malformed or creative answer
+ *      is rejected by the API rather than parsed hopefully at this end.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RESPONSE_SCHEMA = exports.SYSTEM_PROMPT = exports.PROMPT_VERSION = void 0;
+exports.renderUserPrompt = renderUserPrompt;
+exports.PROMPT_VERSION = 'annotate-v3';
+exports.SYSTEM_PROMPT = `You label blocks of an email message so a chat application can decide what to show.
+
+You are given the message split into numbered blocks, in order. Return exactly one label for every
+block index you are given. Never return text, only indices and labels.
+
+LABELS
+
+NEW_CONTENT
+  Text the sender wrote themselves in this message. This is the default when unsure between
+  NEW_CONTENT and anything else.
+
+INLINE_QUOTE
+  Old text the sender quoted deliberately in order to answer it here, point by point. The giveaway
+  is that the sender's own new text appears after it and responds to it. Inline quotes must stay
+  visible, because the answer beside them is meaningless without them.
+
+REPLY_HISTORY
+  A copy of an earlier message that the sender's mail client attached automatically. Usually the
+  whole previous message, sitting at the end, introduced by an attribution line such as
+  "On <date>, <person> wrote:", or "-----Original Message-----", or an embedded
+  From:/Sent:/To:/Subject: block. This is the only label whose content gets hidden.
+
+FORWARDED_CONTENT
+  A message the sender forwarded on purpose, usually after "Begin forwarded message:" or
+  "---------- Forwarded message ----------". It is content the sender meant to pass on, so it is
+  kept and collapsed, never hidden.
+
+SIGNATURE
+  The sender's own sign-off block: name, role, company, phone, links, "Sent from my phone".
+
+DISCLAIMER
+  Boilerplate attached by a system, gateway or template independently of what the sender wrote:
+  confidentiality and liability notices, virus scanning notices, company registration and VAT
+  numbers, registered office addresses, unsubscribe and preference footers, "powered by" branding,
+  and standing promotional footers that repeat on every notification from a service.
+
+  Guard: if the boilerplate is the whole message, label it NEW_CONTENT. A newsletter is not a
+  disclaimer with nothing attached to it. When there is no other content, the boilerplate is the
+  content.
+
+UNKNOWN
+  Use when you genuinely cannot tell. Prefer this over guessing REPLY_HISTORY.
+
+HOW TO DECIDE
+
+The important distinction is INLINE_QUOTE versus REPLY_HISTORY. Both are old text. The difference
+is purpose:
+
+  - If quoted text is followed by the sender's answer to it, and especially if this alternates
+    several times, it is INLINE_QUOTE.
+  - If quoted text runs to the end of the message as one uninterrupted copy of the previous mail,
+    it is REPLY_HISTORY.
+
+Quote characters such as ">" are a hint, not proof. Some clients embed the previous message with no
+quote markers at all, as plain paragraphs after an attribution line: that is still REPLY_HISTORY.
+Some senders paste text that merely looks like an email; that is NEW_CONTENT.
+
+QUOTE DEPTH
+
+Each block is marked with its quote depth, exactly as the message shows it. Depth 0 means the block
+was not quoted. Depth 1 means it was quoted once, depth 2 means it was quoted inside a quote, and so
+on. Depth is structural evidence, not a verdict.
+
+Long threads nest: depth 1 is the message being replied to, depth 2 is the one before that. In a
+nested thread the sender's own new text is almost always at depth 0, and everything at depth 1 or
+deeper is history, including greetings, thanks, questions and sign-offs. Text reading like a fresh
+personal message is still REPLY_HISTORY if it is quoted, because someone wrote it in an earlier
+message, not in this one.
+
+REGION WINS OVER CONTENT TYPE
+
+A message is made of regions: what the sender wrote now, history their client attached, material
+they forwarded on purpose. Signatures, greetings and legal footers exist inside all of them.
+
+Label every block by the region it belongs to, never by what the text looks like. The question is
+always "which message does this block belong to?".
+
+  - A signature belonging to this sender, in this message, is SIGNATURE.
+  - A signature inside quoted history is REPLY_HISTORY. It was someone else's sign-off, written in
+    an earlier message.
+  - A legal disclaimer inside a forwarded message is FORWARDED_CONTENT.
+  - A "--" delimiter inside a quote is part of that quote.
+
+This applies to every footer type, not just signatures. A confidentiality notice sitting inside a
+forward is FORWARDED_CONTENT, not DISCLAIMER.
+
+COST OF ERRORS
+
+Marking the sender's own words as REPLY_HISTORY hides them from the reader, who may never find out
+what was said. Leaving old text visible is only untidy. Treat the first mistake as ten times worse
+than the second. When you are torn between NEW_CONTENT and REPLY_HISTORY, choose NEW_CONTENT. When
+you are torn between INLINE_QUOTE and REPLY_HISTORY, choose INLINE_QUOTE.
+
+CONFIDENCE
+
+Give each label a confidence between 0 and 1. Use below 0.7 whenever the block is ambiguous, short,
+or could plausibly take another label.`;
+exports.RESPONSE_SCHEMA = {
+    type: 'object',
+    additionalProperties: false,
+    required: ['labels'],
+    properties: {
+        labels: {
+            type: 'array',
+            items: {
+                type: 'object',
+                additionalProperties: false,
+                required: ['index', 'label', 'confidence'],
+                properties: {
+                    index: { type: 'integer', description: 'The block number being labelled.' },
+                    label: {
+                        type: 'string',
+                        enum: ['NEW_CONTENT', 'INLINE_QUOTE', 'REPLY_HISTORY', 'FORWARDED_CONTENT', 'SIGNATURE', 'DISCLAIMER', 'UNKNOWN']
+                    },
+                    confidence: { type: 'number' }
+                }
+            }
+        }
+    }
+};
+/**
+ * Render the user message.
+ *
+ * Deliberately minimal context: block text, quote depth, and whether the message is a reply. No
+ * subject, no addresses, no headers. Those would help a little and would send noticeably more
+ * personal data to a third party, which is the wrong trade for a marginal accuracy gain.
+ *
+ * Quote depth is shown because a human annotator reading the raw message would see the ">" markers
+ * and the visual nesting. Hiding it made the first version of this prompt label quoted greetings
+ * inside a five deep Estonian thread as freshly written text, which is exactly the mistake the
+ * depth marker prevents.
+ */
+function renderUserPrompt(blocks, isReply) {
+    const lines = [];
+    lines.push(isReply ? 'This message is a reply to an earlier message.' : 'This message is not marked as a reply.');
+    lines.push(`It has ${blocks.length} block(s), in the order they appear. Label every index listed below.`);
+    lines.push('');
+    for (const block of blocks) {
+        const depth = block.quoteDepth > 0 ? `, quoted depth ${block.quoteDepth}` : ', not quoted';
+        lines.push(`[${block.index}${depth}${block.truncated ? ', truncated' : ''}]`);
+        lines.push(block.text);
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=prompt.js.map

package/dist/annotate/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../src/annotate/prompt.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;AAkJH,4CAYC;AA5JY,QAAA,cAAc,GAAG,aAAa,CAAC;AAE/B,QAAA,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uCAgGU,CAAC;AAE3B,QAAA,eAAe,GAA4B;IACpD,IAAI,EAAE,QAAQ;IACd,oBAAoB,EAAE,KAAK;IAC3B,QAAQ,EAAE,CAAC,QAAQ,CAAC;IACpB,UAAU,EAAE;QACR,MAAM,EAAE;YACJ,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACH,IAAI,EAAE,QAAQ;gBACd,oBAAoB,EAAE,KAAK;gBAC3B,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC;gBAC1C,UAAU,EAAE;oBACR,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;oBAC3E,KAAK,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,mBAAmB,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,CAAC;qBACpH;oBACD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBACjC;aACJ;SACJ;KACJ;CACJ,CAAC;AAUF;;;;;;;;;;;GAWG;AACH,SAAgB,gBAAgB,CAAC,MAA8B,EAAE,OAAgB;IAC7E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,gDAAgD,CAAC,CAAC,CAAC,wCAAwC,CAAC,CAAC;IAClH,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,MAAM,sEAAsE,CAAC,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;QAC3F,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC"}

package/dist/annotate/store.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Annotation persistence.
+ *
+ * Every label carries its source, the model and prompt version that produced it, and the
+ * deterministic baseline's verdict on the same block captured at the same moment. Storing the
+ * baseline alongside is what makes a run reproducible as evidence: a later rule change cannot
+ * silently rewrite what the disagreement was.
+ */
+import { type CorpusStore } from '../corpus/store';
+import { type Label, type Reason } from '../types';
+export interface AnnotationRow {
+    path: string;
+    representation: 'text' | 'html';
+    blockIndex: number;
+    mimePart: string;
+    blockStart: number;
+    blockEnd: number;
+    charCount: number;
+    source: 'llm' | 'human' | 'weak' | 'distant';
+    label: Label;
+    confidence: number;
+    baselineLabel: Label | null;
+    baselineConfidence: number | null;
+    baselineReasons: readonly Reason[];
+}
+export interface AnnotationRunInfo {
+    source: string;
+    model: string | null;
+    promptVersion: string | null;
+    notes?: string;
+}
+export declare function startAnnotationRun(store: CorpusStore, info: AnnotationRunInfo): number;
+export declare function finishAnnotationRun(store: CorpusStore, runId: number, counts: {
+    messages: number;
+    blocks: number;
+    failures: number;
+    promptTokens: number;
+    completionTokens: number;
+}): void;
+export declare function writeAnnotations(store: CorpusStore, runId: number, rows: readonly AnnotationRow[]): void;
+/** Paths already annotated by this source and model, so a run can resume. */
+export declare function annotatedPaths(store: CorpusStore, source: string, model: string | null): Set<string>;
+//# sourceMappingURL=store.d.ts.map

package/dist/annotate/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAqB,KAAK,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AAEtE,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,KAAK,GAAG,IAAI,CAAC;IAC5B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAYtF;AAED,wBAAgB,mBAAmB,CAC/B,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,GAC/G,IAAI,CAaN;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,aAAa,EAAE,GAAG,IAAI,CA6BxG;AAED,6EAA6E;AAC7E,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CASpG"}

package/dist/annotate/store.js

@@ -0,0 +1,46 @@
+"use strict";
+/**
+ * Annotation persistence.
+ *
+ * Every label carries its source, the model and prompt version that produced it, and the
+ * deterministic baseline's verdict on the same block captured at the same moment. Storing the
+ * baseline alongside is what makes a run reproducible as evidence: a later rule change cannot
+ * silently rewrite what the disagreement was.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startAnnotationRun = startAnnotationRun;
+exports.finishAnnotationRun = finishAnnotationRun;
+exports.writeAnnotations = writeAnnotations;
+exports.annotatedPaths = annotatedPaths;
+const types_1 = require("../types");
+function startAnnotationRun(store, info) {
+    const result = store.run(`INSERT INTO annotation_runs (source, model, prompt_version, extractor_version, started_at, notes)
+         VALUES (?, ?, ?, ?, ?, ?)`, info.source, info.model, info.promptVersion, types_1.EXTRACTOR_VERSION, Date.now(), info.notes ?? null);
+    return Number(result.lastInsertRowid);
+}
+function finishAnnotationRun(store, runId, counts) {
+    store.run(`UPDATE annotation_runs
+         SET finished_at = ?, messages = ?, blocks = ?, failures = ?, prompt_tokens = ?, completion_tokens = ?
+         WHERE id = ?`, Date.now(), counts.messages, counts.blocks, counts.failures, counts.promptTokens, counts.completionTokens, runId);
+}
+function writeAnnotations(store, runId, rows) {
+    if (!rows.length) {
+        return;
+    }
+    store.transaction(() => {
+        for (const row of rows) {
+            store.run(`INSERT INTO annotations (
+                    run_id, path, representation, block_index, mime_part, block_start, block_end, char_count,
+                    source, label, confidence, baseline_label, baseline_conf, baseline_reasons, agrees
+                 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, runId, row.path, row.representation, row.blockIndex, row.mimePart, row.blockStart, row.blockEnd, row.charCount, row.source, row.label, row.confidence, row.baselineLabel, row.baselineConfidence, row.baselineReasons.join(','), row.baselineLabel === row.label ? 1 : 0);
+        }
+    });
+}
+/** Paths already annotated by this source and model, so a run can resume. */
+function annotatedPaths(store, source, model) {
+    const rows = model
+        ? store.query('SELECT DISTINCT a.path AS path FROM annotations a JOIN annotation_runs r ON r.id = a.run_id WHERE a.source = ? AND r.model = ?', source, model)
+        : store.query('SELECT DISTINCT path FROM annotations WHERE source = ?', source);
+    return new Set(rows.map(row => row.path));
+}
+//# sourceMappingURL=store.js.map

package/dist/annotate/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/annotate/store.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;AA4BH,gDAYC;AAED,kDAiBC;AAED,4CA6BC;AAGD,wCASC;AAnGD,oCAAsE;AAyBtE,SAAgB,kBAAkB,CAAC,KAAkB,EAAE,IAAuB;IAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CACpB;mCAC2B,EAC3B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,aAAa,EAClB,yBAAiB,EACjB,IAAI,CAAC,GAAG,EAAE,EACV,IAAI,CAAC,KAAK,IAAI,IAAI,CACrB,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,SAAgB,mBAAmB,CAC/B,KAAkB,EAClB,KAAa,EACb,MAA8G;IAE9G,KAAK,CAAC,GAAG,CACL;;sBAEc,EACd,IAAI,CAAC,GAAG,EAAE,EACV,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,gBAAgB,EACvB,KAAK,CACR,CAAC;AACN,CAAC;AAED,SAAgB,gBAAgB,CAAC,KAAkB,EAAE,KAAa,EAAE,IAA8B;IAC9F,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACf,OAAO;IACX,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,KAAK,CAAC,GAAG,CACL;;;wEAGwD,EACxD,KAAK,EACL,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,cAAc,EAClB,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,MAAM,EACV,GAAG,CAAC,KAAK,EACT,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,aAAa,EACjB,GAAG,CAAC,kBAAkB,EACtB,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC1C,CAAC;QACN,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED,6EAA6E;AAC7E,SAAgB,cAAc,CAAC,KAAkB,EAAE,MAAc,EAAE,KAAoB;IACnF,MAAM,IAAI,GAAG,KAAK;QACd,CAAC,CAAC,KAAK,CAAC,KAAK,CACP,gIAAgI,EAChI,MAAM,EACN,KAAK,CACR;QACH,CAAC,CAAC,KAAK,CAAC,KAAK,CAAmB,wDAAwD,EAAE,MAAM,CAAC,CAAC;IACtG,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC"}

package/dist/canonical/html.d.ts

@@ -0,0 +1,33 @@
+/**
+ * HTML helpers used by the corpus scanner.
+ *
+ * Full DOM based block segmentation belongs to the extraction pipeline and is intentionally not
+ * done here. A corpus pass over 320k files runs this code once per message, and parsing every
+ * newsletter into a DOM to compute aggregate statistics would dominate the whole scan. The client
+ * hints this module looks for are literal class and id names, so a scan over the source is both
+ * cheaper and more faithful than a DOM query, which would silently normalise malformed markup.
+ */
+/** Decode HTML entities. Unknown entities are left as written rather than dropped. */
+export declare function decodeEntities(input: string): string;
+/**
+ * Cheap text projection of an HTML body.
+ *
+ * Good enough for language estimation, marker detection and length statistics. It is not a
+ * replacement for the extraction pipeline's DOM walk, and it deliberately keeps blank lines so that
+ * paragraph structure survives.
+ */
+export declare function stripTagsToText(html: string): string;
+/** Structural hints found in an HTML body, keyed by the id used in html-clients.json. */
+export interface HtmlHints {
+    ids: string[];
+    hasBlockquote: boolean;
+    blockquoteCount: number;
+    hasTable: boolean;
+    tableCount: number;
+    hasImage: boolean;
+    /** True when the markup contains an unclosed tag pattern that suggests malformed HTML. */
+    suspectMalformed: boolean;
+}
+/** Detect client structure hints without building a DOM. */
+export declare function detectHtmlHints(html: string): HtmlHints;
+//# sourceMappingURL=html.d.ts.map

package/dist/canonical/html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../src/canonical/html.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAgDH,sFAAsF;AACtF,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAoBpD;AAQD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAgBpD;AAED,yFAAyF;AACzF,MAAM,WAAW,SAAS;IACtB,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,0FAA0F;IAC1F,gBAAgB,EAAE,OAAO,CAAC;CAC7B;AAyBD,4DAA4D;AAC5D,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAqBvD"}

package/dist/canonical/html.js

@@ -0,0 +1,156 @@
+"use strict";
+/**
+ * HTML helpers used by the corpus scanner.
+ *
+ * Full DOM based block segmentation belongs to the extraction pipeline and is intentionally not
+ * done here. A corpus pass over 320k files runs this code once per message, and parsing every
+ * newsletter into a DOM to compute aggregate statistics would dominate the whole scan. The client
+ * hints this module looks for are literal class and id names, so a scan over the source is both
+ * cheaper and more faithful than a DOM query, which would silently normalise malformed markup.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeEntities = decodeEntities;
+exports.stripTagsToText = stripTagsToText;
+exports.detectHtmlHints = detectHtmlHints;
+/** Entities common in mail. Numeric references are handled separately. */
+const NAMED_ENTITIES = {
+    amp: '&',
+    lt: '<',
+    gt: '>',
+    quot: '"',
+    apos: "'",
+    nbsp: ' ',
+    ndash: '–',
+    mdash: '—',
+    lsquo: '‘',
+    rsquo: '’',
+    ldquo: '“',
+    rdquo: '”',
+    hellip: '…',
+    bull: '•',
+    middot: '·',
+    laquo: '«',
+    raquo: '»',
+    copy: '©',
+    reg: '®',
+    trade: '™',
+    euro: '€',
+    pound: '£',
+    deg: '°',
+    szlig: 'ß',
+    auml: 'ä',
+    ouml: 'ö',
+    uuml: 'ü',
+    Auml: 'Ä',
+    Ouml: 'Ö',
+    Uuml: 'Ü',
+    aring: 'å',
+    Aring: 'Å',
+    aelig: 'æ',
+    oslash: 'ø',
+    eacute: 'é',
+    egrave: 'è',
+    agrave: 'à',
+    ccedil: 'ç',
+    ntilde: 'ñ',
+    shy: '­',
+    zwnj: '‌',
+    zwj: '‍'
+};
+/** Decode HTML entities. Unknown entities are left as written rather than dropped. */
+function decodeEntities(input) {
+    if (!input.includes('&')) {
+        return input;
+    }
+    return input.replace(/&(#[xX]?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]{1,31});/g, (whole, body) => {
+        if (body.charCodeAt(0) === 35) {
+            const isHex = body[1] === 'x' || body[1] === 'X';
+            const code = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10);
+            if (!Number.isFinite(code) || code < 0 || code > 0x10ffff) {
+                return whole;
+            }
+            try {
+                return String.fromCodePoint(code);
+            }
+            catch {
+                return whole;
+            }
+        }
+        const named = NAMED_ENTITIES[body];
+        return named === undefined ? whole : named;
+    });
+}
+/** Elements whose content is never user visible text. */
+const INVISIBLE = /<(script|style|head|title|noscript|template)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
+/** Elements that introduce a line break in the text projection. */
+const BLOCK_BREAK = /<\/?(p|div|br|tr|li|h[1-6]|table|blockquote|pre|hr|section|article|ul|ol|dd|dt)\b[^>]*>/gi;
+/**
+ * Cheap text projection of an HTML body.
+ *
+ * Good enough for language estimation, marker detection and length statistics. It is not a
+ * replacement for the extraction pipeline's DOM walk, and it deliberately keeps blank lines so that
+ * paragraph structure survives.
+ */
+function stripTagsToText(html) {
+    let text = html.replace(INVISIBLE, ' ');
+    text = text.replace(/<!--[\s\S]*?-->/g, ' ');
+    // Word and Outlook emit downlevel-revealed conditional comments, which are not HTML comments
+    // and survive both the comment strip and the tag strip: <![if !supportLists]>. Left in, they
+    // appear verbatim in the extracted text of every Word authored bullet list.
+    text = text.replace(/<!\[[^\]]*\]>/g, ' ');
+    text = text.replace(BLOCK_BREAK, '\n');
+    text = text.replace(/<[^>]*>/g, '');
+    text = decodeEntities(text);
+    text = text.replace(/\r\n?/g, '\n');
+    // Includes U+00A0, which arrives constantly as &nbsp; and must collapse like a normal space.
+    text = text.replace(/[ \t\u00a0]+/g, ' ');
+    text = text.replace(/ *\n */g, '\n');
+    text = text.replace(/\n{3,}/g, '\n\n');
+    return text.trim();
+}
+const HINT_PATTERNS = [
+    { id: 'gmail_quote', regex: /class="[^"]*\bgmail_quote\b/i },
+    { id: 'gmail_quote_container', regex: /class="[^"]*\bgmail_quote_container\b/i },
+    { id: 'gmail_attr', regex: /class="[^"]*\bgmail_attr\b/i },
+    { id: 'gmail_signature', regex: /class="[^"]*\bgmail_signature\b/i },
+    { id: 'owa_reply_separator', regex: /id="?divRplyFwdMsg/i },
+    { id: 'owa_append_on_send', regex: /id="?appendonsend/i },
+    { id: 'owa_element_to_proof', regex: /class="[^"]*elementToProof/i },
+    { id: 'outlook_mso_normal', regex: /class="?[^"]*\bMsoNormal\b/i },
+    { id: 'outlook_word_section', regex: /\bWordSection1\b/i },
+    { id: 'outlook_nested_x_class', regex: /class="[^"]*\bx_[A-Za-z]/ },
+    { id: 'apple_mail_cite', regex: /<blockquote[^>]+type="?cite/i },
+    { id: 'apple_interchange_newline', regex: /Apple-interchange-newline/i },
+    { id: 'apple_mail_signature', regex: /AppleMailSignature/i },
+    { id: 'thunderbird_moz_cite', regex: /moz-cite-prefix/i },
+    { id: 'thunderbird_signature', regex: /moz-signature/i },
+    { id: 'proton_quote', regex: /protonmail_quote/i },
+    { id: 'yahoo_quoted', regex: /yahoo_quoted/i },
+    { id: 'zimbra_separator', regex: /id="?zwchr/i },
+    { id: 'outlook_border_separator', regex: /border-top:\s*(solid|1px|1\.0pt)/i },
+    { id: 'ms_outlook_mobile', regex: /ms-outlook-mobile|Outlook for (iOS|Android)/i }
+];
+/** Detect client structure hints without building a DOM. */
+function detectHtmlHints(html) {
+    const head = html.length > 262144 ? html.slice(0, 262144) : html;
+    const ids = [];
+    for (const { id, regex } of HINT_PATTERNS) {
+        if (regex.test(head)) {
+            ids.push(id);
+        }
+    }
+    const blockquoteCount = (head.match(/<blockquote\b/gi) || []).length;
+    const tableCount = (head.match(/<table\b/gi) || []).length;
+    return {
+        ids,
+        hasBlockquote: blockquoteCount > 0,
+        blockquoteCount,
+        hasTable: tableCount > 0,
+        tableCount,
+        hasImage: /<img\b/i.test(head),
+        // A body element inside a table cell, or a closing tag with no opener nearby, is a decent
+        // proxy for markup that needs a forgiving parser.
+        suspectMalformed: /<\/(td|tr|table)>\s*<\/(html|body)>/i.test(head) || blockquoteCount !== (head.match(/<\/blockquote>/gi) || []).length
+    };
+}
+//# sourceMappingURL=html.js.map

package/dist/canonical/html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.js","sourceRoot":"","sources":["../../src/canonical/html.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAiDH,wCAoBC;AAeD,0CAgBC;AAsCD,0CAqBC;AA7JD,0EAA0E;AAC1E,MAAM,cAAc,GAA2B;IAC3C,GAAG,EAAE,GAAG;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,GAAG;IACX,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,GAAG,EAAE,GAAG;IACR,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;CACX,CAAC;AAEF,sFAAsF;AACtF,SAAgB,cAAc,CAAC,KAAa;IACxC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,OAAO,CAAC,mDAAmD,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;QAC9F,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;YACjD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;gBACxD,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,IAAI,CAAC;gBACD,OAAO,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/C,CAAC,CAAC,CAAC;AACP,CAAC;AAED,yDAAyD;AACzD,MAAM,SAAS,GAAG,yEAAyE,CAAC;AAE5F,mEAAmE;AACnE,MAAM,WAAW,GAAG,2FAA2F,CAAC;AAEhH;;;;;;GAMG;AACH,SAAgB,eAAe,CAAC,IAAY;IACxC,IAAI,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IAC7C,6FAA6F;IAC7F,6FAA6F;IAC7F,4EAA4E;IAC5E,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACpC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACpC,6FAA6F;IAC7F,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAcD,MAAM,aAAa,GAA6C;IAC5D,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,8BAA8B,EAAE;IAC5D,EAAE,EAAE,EAAE,uBAAuB,EAAE,KAAK,EAAE,wCAAwC,EAAE;IAChF,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,6BAA6B,EAAE;IAC1D,EAAE,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,kCAAkC,EAAE;IACpE,EAAE,EAAE,EAAE,qBAAqB,EAAE,KAAK,EAAE,qBAAqB,EAAE;IAC3D,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,oBAAoB,EAAE;IACzD,EAAE,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,6BAA6B,EAAE;IACpE,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE,6BAA6B,EAAE;IAClE,EAAE,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,mBAAmB,EAAE;IAC1D,EAAE,EAAE,EAAE,wBAAwB,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnE,EAAE,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,8BAA8B,EAAE;IAChE,EAAE,EAAE,EAAE,2BAA2B,EAAE,KAAK,EAAE,4BAA4B,EAAE;IACxE,EAAE,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,qBAAqB,EAAE;IAC5D,EAAE,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,kBAAkB,EAAE;IACzD,EAAE,EAAE,EAAE,uBAAuB,EAAE,KAAK,EAAE,gBAAgB,EAAE;IACxD,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,mBAAmB,EAAE;IAClD,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE;IAC9C,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,aAAa,EAAE;IAChD,EAAE,EAAE,EAAE,0BAA0B,EAAE,KAAK,EAAE,mCAAmC,EAAE;IAC9E,EAAE,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,8CAA8C,EAAE;CACrF,CAAC;AAEF,4DAA4D;AAC5D,SAAgB,eAAe,CAAC,IAAY;IACxC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,aAAa,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACL,CAAC;IACD,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3D,OAAO;QACH,GAAG;QACH,aAAa,EAAE,eAAe,GAAG,CAAC;QAClC,eAAe;QACf,QAAQ,EAAE,UAAU,GAAG,CAAC;QACxB,UAAU;QACV,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9B,0FAA0F;QAC1F,kDAAkD;QAClD,gBAAgB,EAAE,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,eAAe,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM;KAC3I,CAAC;AACN,CAAC"}

package/dist/canonical/text.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Stage 2: plain text canonicalisation.
+ *
+ * Offsets in the rest of the pipeline are character offsets into the canonical string produced
+ * here, not byte offsets into the raw message. Traceability back to the file is via the owning MIME
+ * part's byte range. That split is deliberate: transfer decoding, charset conversion and line
+ * ending normalisation all change lengths, and maintaining a byte level offset map through three
+ * lossy transforms would be fragile for no practical gain.
+ *
+ * What this module does NOT do is unwrap hard wrapped lines. Unwrapping changes offsets and can
+ * merge a quoted line with a new one, which is exactly the mistake that loses content. Wrapping is
+ * recorded as a per line property and resolved later, in the matching projection only.
+ */
+/** A single canonical line with its quote depth stripped out. */
+export interface TextLine {
+    /** Zero based index into the line array. */
+    index: number;
+    /** Character offset of the line start within the canonical text. */
+    start: number;
+    /** Character offset just past the line content, excluding the newline. */
+    end: number;
+    /** The line exactly as it appears in the canonical text, including any quote prefix. */
+    raw: string;
+    /** The line with its quote prefix removed. */
+    content: string;
+    /** Number of leading quote markers. Zero for unquoted lines. */
+    depth: number;
+    /** True when the line has no non whitespace content after prefix removal. */
+    blank: boolean;
+    /** True when this unquoted line was re-attached as the wrapped tail of the quote above it. */
+    wrapContinuation?: boolean;
+}
+export interface CanonicalText {
+    /** Normalised text. All offsets in the pipeline refer to this string. */
+    text: string;
+    lines: TextLine[];
+}
+export type ReplyStyle = 'none' | 'top' | 'bottom' | 'interleaved' | 'top-and-trailing' | 'quote-only';
+export interface QuoteProfile {
+    maxDepth: number;
+    quotedLines: number;
+    totalLines: number;
+    /** Share of non blank lines that carry a quote prefix. */
+    quotedRatio: number;
+    style: ReplyStyle;
+    /** Non blank unquoted lines before the first quoted line. */
+    contentBefore: number;
+    /** Non blank unquoted lines between the first and last quoted line. */
+    contentBetween: number;
+    /** Non blank unquoted lines after the last quoted line. */
+    contentAfter: number;
+}
+/** Split a line into its quote depth and remaining content. */
+export declare function splitQuotePrefix(line: string): {
+    depth: number;
+    content: string;
+};
+/**
+ * Normalise decoded part text into the canonical form used by every later stage.
+ *
+ * Line endings become LF, a BOM is dropped, NUL bytes and lone carriage returns are removed, and
+ * the result is NFC normalised so that visually identical text compares equal during history
+ * matching. Nothing else is altered: no trimming, no whitespace collapsing, no unwrapping.
+ */
+export declare function canonicaliseText(input: string): CanonicalText;
+/**
+ * Describe how quoting is laid out in a plain text body.
+ *
+ * The style categories are positional only. They describe where unquoted content sits relative to
+ * quoted content and make no claim about intent, because a trailing block after a quote is just as
+ * likely to be a signature as a genuine bottom posted answer. Deciding that is the classifier's
+ * job, not this function's.
+ */
+export declare function profileQuoting(lines: readonly TextLine[]): QuoteProfile;
+//# sourceMappingURL=text.d.ts.map

package/dist/canonical/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../src/canonical/text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,iEAAiE;AACjE,MAAM,WAAW,QAAQ;IACrB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,GAAG,EAAE,MAAM,CAAC;IACZ,wFAAwF;IACxF,GAAG,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,KAAK,EAAE,OAAO,CAAC;IACf,8FAA8F;IAC9F,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC1B,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,EAAE,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,aAAa,GAAG,kBAAkB,GAAG,YAAY,CAAC;AAEvG,MAAM,WAAW,YAAY;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,CAAC;IAClB,6DAA6D;IAC7D,aAAa,EAAE,MAAM,CAAC;IACtB,uEAAuE;IACvE,cAAc,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,YAAY,EAAE,MAAM,CAAC;CACxB;AAYD,+DAA+D;AAC/D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAajF;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAwC7D;AA+CD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,GAAG,YAAY,CA0EvE"}