@postalsys/email-content 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +288 -0
  3. package/dist/.tsbuildinfo +1 -0
  4. package/dist/analyze.d.ts +91 -0
  5. package/dist/analyze.d.ts.map +1 -0
  6. package/dist/analyze.js +274 -0
  7. package/dist/analyze.js.map +1 -0
  8. package/dist/annotate/client.d.ts +64 -0
  9. package/dist/annotate/client.d.ts.map +1 -0
  10. package/dist/annotate/client.js +139 -0
  11. package/dist/annotate/client.js.map +1 -0
  12. package/dist/annotate/llm.d.ts +46 -0
  13. package/dist/annotate/llm.d.ts.map +1 -0
  14. package/dist/annotate/llm.js +107 -0
  15. package/dist/annotate/llm.js.map +1 -0
  16. package/dist/annotate/prompt.d.ts +38 -0
  17. package/dist/annotate/prompt.d.ts.map +1 -0
  18. package/dist/annotate/prompt.js +165 -0
  19. package/dist/annotate/prompt.js.map +1 -0
  20. package/dist/annotate/store.d.ts +43 -0
  21. package/dist/annotate/store.d.ts.map +1 -0
  22. package/dist/annotate/store.js +46 -0
  23. package/dist/annotate/store.js.map +1 -0
  24. package/dist/canonical/html.d.ts +33 -0
  25. package/dist/canonical/html.d.ts.map +1 -0
  26. package/dist/canonical/html.js +156 -0
  27. package/dist/canonical/html.js.map +1 -0
  28. package/dist/canonical/text.d.ts +75 -0
  29. package/dist/canonical/text.d.ts.map +1 -0
  30. package/dist/canonical/text.js +217 -0
  31. package/dist/canonical/text.js.map +1 -0
  32. package/dist/classify/confidence.d.ts +38 -0
  33. package/dist/classify/confidence.d.ts.map +1 -0
  34. package/dist/classify/confidence.js +59 -0
  35. package/dist/classify/confidence.js.map +1 -0
  36. package/dist/classify/reconcile.d.ts +52 -0
  37. package/dist/classify/reconcile.d.ts.map +1 -0
  38. package/dist/classify/reconcile.js +100 -0
  39. package/dist/classify/reconcile.js.map +1 -0
  40. package/dist/classify/rules.d.ts +51 -0
  41. package/dist/classify/rules.d.ts.map +1 -0
  42. package/dist/classify/rules.js +648 -0
  43. package/dist/classify/rules.js.map +1 -0
  44. package/dist/cli/annotate.d.ts +23 -0
  45. package/dist/cli/annotate.d.ts.map +1 -0
  46. package/dist/cli/annotate.js +328 -0
  47. package/dist/cli/annotate.js.map +1 -0
  48. package/dist/cli/args.d.ts +17 -0
  49. package/dist/cli/args.d.ts.map +1 -0
  50. package/dist/cli/args.js +72 -0
  51. package/dist/cli/args.js.map +1 -0
  52. package/dist/cli/benchmark.d.ts +13 -0
  53. package/dist/cli/benchmark.d.ts.map +1 -0
  54. package/dist/cli/benchmark.js +172 -0
  55. package/dist/cli/benchmark.js.map +1 -0
  56. package/dist/cli/corpus.d.ts +9 -0
  57. package/dist/cli/corpus.d.ts.map +1 -0
  58. package/dist/cli/corpus.js +152 -0
  59. package/dist/cli/corpus.js.map +1 -0
  60. package/dist/cli/disagreements.d.ts +14 -0
  61. package/dist/cli/disagreements.d.ts.map +1 -0
  62. package/dist/cli/disagreements.js +133 -0
  63. package/dist/cli/disagreements.js.map +1 -0
  64. package/dist/cli/evaluate.d.ts +15 -0
  65. package/dist/cli/evaluate.d.ts.map +1 -0
  66. package/dist/cli/evaluate.js +215 -0
  67. package/dist/cli/evaluate.js.map +1 -0
  68. package/dist/cli/extract.d.ts +10 -0
  69. package/dist/cli/extract.d.ts.map +1 -0
  70. package/dist/cli/extract.js +139 -0
  71. package/dist/cli/extract.js.map +1 -0
  72. package/dist/cli/main.d.ts +9 -0
  73. package/dist/cli/main.d.ts.map +1 -0
  74. package/dist/cli/main.js +147 -0
  75. package/dist/cli/main.js.map +1 -0
  76. package/dist/corpus/analyze.d.ts +66 -0
  77. package/dist/corpus/analyze.d.ts.map +1 -0
  78. package/dist/corpus/analyze.js +214 -0
  79. package/dist/corpus/analyze.js.map +1 -0
  80. package/dist/corpus/migrations/index.d.ts +14 -0
  81. package/dist/corpus/migrations/index.d.ts.map +1 -0
  82. package/dist/corpus/migrations/index.js +158 -0
  83. package/dist/corpus/migrations/index.js.map +1 -0
  84. package/dist/corpus/sample.d.ts +66 -0
  85. package/dist/corpus/sample.d.ts.map +1 -0
  86. package/dist/corpus/sample.js +98 -0
  87. package/dist/corpus/sample.js.map +1 -0
  88. package/dist/corpus/scanner.d.ts +79 -0
  89. package/dist/corpus/scanner.d.ts.map +1 -0
  90. package/dist/corpus/scanner.js +256 -0
  91. package/dist/corpus/scanner.js.map +1 -0
  92. package/dist/corpus/stats.d.ts +76 -0
  93. package/dist/corpus/stats.d.ts.map +1 -0
  94. package/dist/corpus/stats.js +240 -0
  95. package/dist/corpus/stats.js.map +1 -0
  96. package/dist/corpus/store.d.ts +67 -0
  97. package/dist/corpus/store.d.ts.map +1 -0
  98. package/dist/corpus/store.js +196 -0
  99. package/dist/corpus/store.js.map +1 -0
  100. package/dist/corpus/thread-index.d.ts +31 -0
  101. package/dist/corpus/thread-index.d.ts.map +1 -0
  102. package/dist/corpus/thread-index.js +182 -0
  103. package/dist/corpus/thread-index.js.map +1 -0
  104. package/dist/corpus/worker.d.ts +21 -0
  105. package/dist/corpus/worker.d.ts.map +1 -0
  106. package/dist/corpus/worker.js +74 -0
  107. package/dist/corpus/worker.js.map +1 -0
  108. package/dist/detect/client.d.ts +28 -0
  109. package/dist/detect/client.d.ts.map +1 -0
  110. package/dist/detect/client.js +121 -0
  111. package/dist/detect/client.js.map +1 -0
  112. package/dist/detect/disclaimer.d.ts +12 -0
  113. package/dist/detect/disclaimer.d.ts.map +1 -0
  114. package/dist/detect/disclaimer.js +28 -0
  115. package/dist/detect/disclaimer.js.map +1 -0
  116. package/dist/detect/forward-header.d.ts +10 -0
  117. package/dist/detect/forward-header.d.ts.map +1 -0
  118. package/dist/detect/forward-header.js +35 -0
  119. package/dist/detect/forward-header.js.map +1 -0
  120. package/dist/detect/patterns/disclaimers.json +113 -0
  121. package/dist/detect/patterns/forward-headers.json +40 -0
  122. package/dist/detect/patterns/html-clients.json +157 -0
  123. package/dist/detect/patterns/quoted-headers.json +41 -0
  124. package/dist/detect/patterns/reply-headers.json +139 -0
  125. package/dist/detect/patterns/signatures.json +58 -0
  126. package/dist/detect/patterns.d.ts +43 -0
  127. package/dist/detect/patterns.d.ts.map +1 -0
  128. package/dist/detect/patterns.js +125 -0
  129. package/dist/detect/patterns.js.map +1 -0
  130. package/dist/detect/reply-header.d.ts +44 -0
  131. package/dist/detect/reply-header.d.ts.map +1 -0
  132. package/dist/detect/reply-header.js +134 -0
  133. package/dist/detect/reply-header.js.map +1 -0
  134. package/dist/detect/signature.d.ts +28 -0
  135. package/dist/detect/signature.d.ts.map +1 -0
  136. package/dist/detect/signature.js +77 -0
  137. package/dist/detect/signature.js.map +1 -0
  138. package/dist/detect/traffic.d.ts +26 -0
  139. package/dist/detect/traffic.d.ts.map +1 -0
  140. package/dist/detect/traffic.js +98 -0
  141. package/dist/detect/traffic.js.map +1 -0
  142. package/dist/eval/metrics.d.ts +87 -0
  143. package/dist/eval/metrics.d.ts.map +1 -0
  144. package/dist/eval/metrics.js +185 -0
  145. package/dist/eval/metrics.js.map +1 -0
  146. package/dist/index.d.ts +54 -0
  147. package/dist/index.d.ts.map +1 -0
  148. package/dist/index.js +163 -0
  149. package/dist/index.js.map +1 -0
  150. package/dist/match/history.d.ts +62 -0
  151. package/dist/match/history.d.ts.map +1 -0
  152. package/dist/match/history.js +128 -0
  153. package/dist/match/history.js.map +1 -0
  154. package/dist/match/normalize.d.ts +33 -0
  155. package/dist/match/normalize.d.ts.map +1 -0
  156. package/dist/match/normalize.js +63 -0
  157. package/dist/match/normalize.js.map +1 -0
  158. package/dist/match/shingle.d.ts +44 -0
  159. package/dist/match/shingle.d.ts.map +1 -0
  160. package/dist/match/shingle.js +78 -0
  161. package/dist/match/shingle.js.map +1 -0
  162. package/dist/mime/charset.d.ts +33 -0
  163. package/dist/mime/charset.d.ts.map +1 -0
  164. package/dist/mime/charset.js +118 -0
  165. package/dist/mime/charset.js.map +1 -0
  166. package/dist/mime/headers.d.ts +32 -0
  167. package/dist/mime/headers.d.ts.map +1 -0
  168. package/dist/mime/headers.js +127 -0
  169. package/dist/mime/headers.js.map +1 -0
  170. package/dist/mime/model.d.ts +111 -0
  171. package/dist/mime/model.d.ts.map +1 -0
  172. package/dist/mime/model.js +17 -0
  173. package/dist/mime/model.js.map +1 -0
  174. package/dist/mime/parse.d.ts +27 -0
  175. package/dist/mime/parse.d.ts.map +1 -0
  176. package/dist/mime/parse.js +343 -0
  177. package/dist/mime/parse.js.map +1 -0
  178. package/dist/segment/features.d.ts +29 -0
  179. package/dist/segment/features.d.ts.map +1 -0
  180. package/dist/segment/features.js +102 -0
  181. package/dist/segment/features.js.map +1 -0
  182. package/dist/segment/html-blocks.d.ts +28 -0
  183. package/dist/segment/html-blocks.d.ts.map +1 -0
  184. package/dist/segment/html-blocks.js +596 -0
  185. package/dist/segment/html-blocks.js.map +1 -0
  186. package/dist/segment/text-blocks.d.ts +30 -0
  187. package/dist/segment/text-blocks.d.ts.map +1 -0
  188. package/dist/segment/text-blocks.js +204 -0
  189. package/dist/segment/text-blocks.js.map +1 -0
  190. package/dist/segment/types.d.ts +88 -0
  191. package/dist/segment/types.d.ts.map +1 -0
  192. package/dist/segment/types.js +11 -0
  193. package/dist/segment/types.js.map +1 -0
  194. package/dist/types.d.ts +85 -0
  195. package/dist/types.d.ts.map +1 -0
  196. package/dist/types.js +17 -0
  197. package/dist/types.js.map +1 -0
  198. package/dist/util/hash.d.ts +49 -0
  199. package/dist/util/hash.d.ts.map +1 -0
  200. package/dist/util/hash.js +0 -0
  201. package/dist/util/hash.js.map +1 -0
  202. package/dist/util/lang.d.ts +25 -0
  203. package/dist/util/lang.d.ts.map +1 -0
  204. package/dist/util/lang.js +135 -0
  205. package/dist/util/lang.js.map +1 -0
  206. package/dist/util/worker-pool.d.ts +51 -0
  207. package/dist/util/worker-pool.d.ts.map +1 -0
  208. package/dist/util/worker-pool.js +155 -0
  209. package/dist/util/worker-pool.js.map +1 -0
  210. package/package.json +74 -0
@@ -0,0 +1,648 @@
1
+ "use strict";
2
+ /**
3
+ * Stage 6: rule based sequence classification.
4
+ *
5
+ * Blocks are classified in context, never independently. The rules encode the four sequences that
6
+ * matter, in the order they are resolved:
7
+ *
8
+ * NEW_CONTENT, SIGNATURE, REPLY_HEADER, REPLY_HISTORY top posting, the common case
9
+ * INLINE_QUOTE, NEW_CONTENT, INLINE_QUOTE, NEW_CONTENT interleaved answers
10
+ * NEW_CONTENT, FORWARD_HEADER, FORWARDED_CONTENT forwarding with a comment
11
+ * REPLY_HISTORY, NEW_CONTENT bottom posting
12
+ *
13
+ * The load bearing rule is the anchor requirement. A quote region may only become REPLY_HISTORY
14
+ * when something independent of the quoting itself says that it is history: an attribution line, a
15
+ * client quote container, or a match against a message that already existed. Quote depth alone
16
+ * gives INLINE_QUOTE or UNKNOWN, both of which stay visible.
17
+ *
18
+ * The reason a bare quote is not treated as history is the difference between the two examples in
19
+ * the brief. Quoted questions with answers under them are context the reader needs. A trailing
20
+ * quoted copy of the previous mail is noise. Only the surrounding structure tells them apart.
21
+ */
22
+ Object.defineProperty(exports, "__esModule", { value: true });
23
+ exports.classifyBlocks = classifyBlocks;
24
+ const confidence_1 = require("./confidence");
25
+ /**
26
+ * Evidence ids that mark the first line of a signature rather than its body.
27
+ *
28
+ * A closing greeting or a "sent from my phone" line opens a signature. A phone number or a street
29
+ * address sits inside one.
30
+ */
31
+ const TOP_OF_SIGNATURE = new Set(['signature:best_regards', 'signature:sent_from_device', 'signature:sent_from_localised', 'signature:dash_dash_space']);
32
+ /** The signature evidence id on a block, if any. */
33
+ function topSignal(block) {
34
+ for (const evidence of block.evidence) {
35
+ if (evidence.kind === 'signature' && TOP_OF_SIGNATURE.has(evidence.id)) {
36
+ return evidence.id;
37
+ }
38
+ }
39
+ return '';
40
+ }
41
+ /** Highest strength evidence of a given kind attached to a block. */
42
+ function strengthOf(block, kinds) {
43
+ let best = 0;
44
+ for (const evidence of block.evidence) {
45
+ if (kinds.includes(evidence.kind) && evidence.strength > best) {
46
+ best = evidence.strength;
47
+ }
48
+ }
49
+ return best;
50
+ }
51
+ function hasWords(block) {
52
+ return block.features.wordCount > 0 && !block.features.isAllBlank;
53
+ }
54
+ /** A quoted block, in either representation. */
55
+ function isQuote(block) {
56
+ return block.kind === 'quote' || block.features.quoteDepth > 0 || block.features.blockquoteDepth > 0;
57
+ }
58
+ /** Client generated boilerplate that introduces a quote. Not authored by the sender. */
59
+ function isAnchorBlock(block) {
60
+ return block.kind === 'reply_header' || block.kind === 'quoted_header_block';
61
+ }
62
+ /** Group blocks into alternating quoted and unquoted runs, ignoring blank blocks. */
63
+ function findRuns(blocks) {
64
+ const runs = [];
65
+ for (const block of blocks) {
66
+ if (block.features.isAllBlank) {
67
+ continue;
68
+ }
69
+ const quoted = isQuote(block) || isAnchorBlock(block);
70
+ const last = runs[runs.length - 1];
71
+ if (last && last.quoted === quoted) {
72
+ last.end = block.index;
73
+ }
74
+ else {
75
+ runs.push({ start: block.index, end: block.index, quoted });
76
+ }
77
+ }
78
+ return runs;
79
+ }
80
+ /**
81
+ * Classify one representation's blocks.
82
+ *
83
+ * Runs in four passes so that later passes can see the whole picture: footers first, because they
84
+ * bound where history can start; then anchors; then regions; then defaults.
85
+ */
86
+ function classifyBlocks(blocks, options = {}) {
87
+ const config = options.config ?? confidence_1.DEFAULT_CONFIDENCE;
88
+ const matched = options.historyMatchedIndices ?? new Set();
89
+ const senderTokens = options.senderTokens ?? [];
90
+ const decisions = blocks.map(block => ({
91
+ block,
92
+ label: 'NEW_CONTENT',
93
+ confidence: 0.95,
94
+ reasons: ['new_content_default']
95
+ }));
96
+ const set = (index, label, confidence, reasons) => {
97
+ const decision = decisions[index];
98
+ if (decision) {
99
+ decision.label = label;
100
+ decision.confidence = confidence;
101
+ decision.reasons = reasons;
102
+ }
103
+ };
104
+ // Pass 1: locate anchors.
105
+ let replyAnchor = -1;
106
+ let replyAnchorStrength = 0;
107
+ let forwardAnchor = -1;
108
+ let forwardAnchorStrength = 0;
109
+ for (const block of blocks) {
110
+ const forward = strengthOf(block, ['forward-anchor']);
111
+ if (forward > 0 && forwardAnchor === -1) {
112
+ forwardAnchor = block.index;
113
+ forwardAnchorStrength = forward;
114
+ }
115
+ const reply = strengthOf(block, ['reply-anchor']);
116
+ if (reply > 0 && replyAnchor === -1) {
117
+ replyAnchor = block.index;
118
+ replyAnchorStrength = reply;
119
+ }
120
+ }
121
+ // A quote container in HTML is an anchor in its own right, but only when something corroborates
122
+ // it.
123
+ //
124
+ // A client that quotes a message emits an attribution line with the container, or nests the
125
+ // quoted text in a blockquote, or both. A container with neither is not reliably a quote
126
+ // boundary: mailing list relays and some composers wrap arbitrary spans in a gmail_quote div,
127
+ // and one message in the working corpus splits a single sentence across the container boundary.
128
+ // Trusting that container would hide half a sentence of genuine content, so an uncorroborated
129
+ // container is left as ordinary evidence and cannot anchor a hide.
130
+ const hasAttribution = blocks.some(block => isAnchorBlock(block) || block.evidence.some(item => item.kind === 'attribution' || item.kind === 'reply-anchor'));
131
+ const hasBlockquote = blocks.some(block => block.features.blockquoteDepth > 0);
132
+ let containerAnchor = -1;
133
+ let containerStrength = 0;
134
+ if (hasAttribution || hasBlockquote) {
135
+ for (const block of blocks) {
136
+ const container = strengthOf(block, ['quote-container']);
137
+ if (container >= config.minAnchorStrength && containerAnchor === -1) {
138
+ containerAnchor = block.index;
139
+ containerStrength = container;
140
+ }
141
+ }
142
+ }
143
+ if (containerAnchor !== -1 && (replyAnchor === -1 || containerAnchor < replyAnchor)) {
144
+ replyAnchor = containerAnchor;
145
+ replyAnchorStrength = Math.max(replyAnchorStrength, containerStrength);
146
+ }
147
+ // Some clients mark the boundary rather than wrapping the quote: Outlook Web emits an empty
148
+ // #divRplyFwdMsg, Zimbra an #zwchr rule, Outlook desktop a bordered element. Everything after
149
+ // such a separator is history, even though none of it is inside a blockquote.
150
+ let separatorAnchor = -1;
151
+ let separatorStrength = 0;
152
+ for (const block of blocks) {
153
+ const strength = strengthOf(block, ['separator']);
154
+ if (strength >= config.minAnchorStrength) {
155
+ separatorAnchor = block.index;
156
+ separatorStrength = strength;
157
+ break;
158
+ }
159
+ }
160
+ // The plain text underscore rule is far too common to trust on its own. It only counts when an
161
+ // embedded header block follows it closely, which is the shape Outlook actually produces.
162
+ if (separatorAnchor === -1) {
163
+ for (const block of blocks) {
164
+ if (block.kind !== 'separator' || strengthOf(block, ['separator']) < 0.6) {
165
+ continue;
166
+ }
167
+ let seen = 0;
168
+ for (let i = block.index + 1; i < blocks.length && seen < 3; i++) {
169
+ const candidate = blocks[i];
170
+ if (candidate.features.isAllBlank) {
171
+ continue;
172
+ }
173
+ seen++;
174
+ if (candidate.kind === 'quoted_header_block') {
175
+ separatorAnchor = block.index;
176
+ separatorStrength = 0.9;
177
+ break;
178
+ }
179
+ }
180
+ if (separatorAnchor !== -1) {
181
+ break;
182
+ }
183
+ }
184
+ }
185
+ // Pass 2: trailing footers, searched only within the region the sender actually authored.
186
+ //
187
+ // The footer scan has to know where history starts, which is why it runs after anchors. In a
188
+ // top posted reply the signature sits between the new text and the quote, so a naive scan
189
+ // backwards from the end of the message would hit the quoted history first and give up before
190
+ // ever reaching the delimiter. Anything after the first anchor belongs to the quoted message
191
+ // and is classified by the region rules instead.
192
+ const firstQuoted = blocks.find(block => isQuote(block) && !block.features.isAllBlank)?.index ?? blocks.length;
193
+ const authoredEnd = Math.min(firstQuoted, replyAnchor === -1 ? blocks.length : replyAnchor, separatorAnchor === -1 ? blocks.length : separatorAnchor, forwardAnchor === -1 ? blocks.length : forwardAnchor);
194
+ // An RFC 3676 delimiter is the one high confidence signature signal. Everything from it to the
195
+ // end of the authored region is the signature.
196
+ for (let i = authoredEnd - 1; i >= 0; i--) {
197
+ const block = blocks[i];
198
+ if (isQuote(block)) {
199
+ break;
200
+ }
201
+ // Match on the delimiter evidence, not on the block kind. In plain text a "-- " line is a
202
+ // block of its own and comes out as kind 'separator', but in HTML it is the first line of a
203
+ // paragraph that also holds the signature body. Requiring 'separator' meant the one high
204
+ // confidence signature signal never fired on HTML at all.
205
+ if (block.evidence.some(item => item.id === 'signature:dash_dash_space' && item.strength >= 0.9)) {
206
+ for (let j = i; j < authoredEnd; j++) {
207
+ set(j, 'SIGNATURE', 0.95, ['dash_dash_delimiter']);
208
+ }
209
+ break;
210
+ }
211
+ }
212
+ // Trailing disclaimers, anywhere in the message.
213
+ //
214
+ // A gateway appends its legal footer to the outgoing message, so it lands after the quoted
215
+ // history, outside the authored region the footer scan above covers. Measured against the
216
+ // reference set, disclaimers sit at a mean relative position of 0.86 and 81 percent of them are
217
+ // in the final quarter of the message, so bounding the scan to the authored region missed
218
+ // essentially all of them.
219
+ //
220
+ // Only blocks still considered authored are converted, so this cannot pull anything out of a
221
+ // history or forwarded region, and DISCLAIMER never hides, it only collapses.
222
+ for (const block of blocks) {
223
+ const decision = decisions[block.index];
224
+ if (!decision || decision.label !== 'NEW_CONTENT') {
225
+ continue;
226
+ }
227
+ const strength = strengthOf(block, ['disclaimer']);
228
+ if (strength >= 0.7) {
229
+ decision.label = 'DISCLAIMER';
230
+ decision.confidence = Math.min(0.95, strength);
231
+ decision.reasons = ['disclaimer_phrase'];
232
+ }
233
+ }
234
+ // Then the signature region.
235
+ //
236
+ // A signature is not one block. In HTML it is typically a run of short blocks at the end of the
237
+ // authored region, one line each: name, then title, then company, then phone, then links. Only
238
+ // one of those carries anything a per block rule would recognise, which is why checking the
239
+ // last block or two found almost nothing.
240
+ //
241
+ // So walk back from the end of the authored region collecting short blocks, stop at the first
242
+ // real paragraph or quote, and accept the run if anything in it looks like a sign-off: contact
243
+ // details, or a closing greeting, or a device phrase. The caps keep an unsigned message from
244
+ // swallowing its own last paragraphs, and SIGNATURE only ever collapses, so the cost of being
245
+ // wrong here is bounded.
246
+ const MAX_SIGNATURE_BLOCKS = 6;
247
+ const MAX_SIGNATURE_WORDS = 60;
248
+ const MAX_BLOCK_WORDS = 25;
249
+ let signatureStart = -1;
250
+ let hasContact = false;
251
+ let hasPhrase = false;
252
+ let hasSenderName = false;
253
+ let phraseStrength = 0;
254
+ let words = 0;
255
+ let collected = 0;
256
+ for (let i = authoredEnd - 1; i >= 0 && collected < MAX_SIGNATURE_BLOCKS; i--) {
257
+ const block = blocks[i];
258
+ const decision = decisions[i];
259
+ if (!decision) {
260
+ continue;
261
+ }
262
+ if (block.features.isAllBlank) {
263
+ continue;
264
+ }
265
+ if (decision.label === 'SIGNATURE') {
266
+ signatureStart = i;
267
+ continue;
268
+ }
269
+ if (isQuote(block) || isAnchorBlock(block) || decision.label === 'DISCLAIMER') {
270
+ break;
271
+ }
272
+ if (block.features.wordCount > MAX_BLOCK_WORDS) {
273
+ break;
274
+ }
275
+ words += block.features.wordCount;
276
+ if (words > MAX_SIGNATURE_WORDS) {
277
+ break;
278
+ }
279
+ if (block.features.hasEmailAddress || block.features.hasPhoneNumber || block.features.hasUrl) {
280
+ hasContact = true;
281
+ }
282
+ const strength = strengthOf(block, ['signature']);
283
+ if (strength > 0) {
284
+ hasPhrase = true;
285
+ phraseStrength = Math.max(phraseStrength, strength);
286
+ }
287
+ if (senderTokens.length && !hasSenderName) {
288
+ const lowered = block.content.toLowerCase();
289
+ if (senderTokens.some(token => lowered.includes(token))) {
290
+ hasSenderName = true;
291
+ }
292
+ }
293
+ signatureStart = i;
294
+ collected++;
295
+ // A closing greeting or a device phrase is the top of a signature, not part of the body
296
+ // above it. Stop here, otherwise the scan keeps walking back over short sentences and
297
+ // eventually swallows the message it was supposed to be signing.
298
+ //
299
+ // Only those two count. Contact lines, postal addresses and the generic shape heuristic are
300
+ // signature *body*, and treating them as the top stopped the scan on the first line it saw,
301
+ // which is why a six line corporate signature was recognised as no signature at all.
302
+ if (TOP_OF_SIGNATURE.has(topSignal(block))) {
303
+ break;
304
+ }
305
+ }
306
+ // Two independent signals, or one very strong one. Requiring only "something looks like a
307
+ // sign-off" labelled a quarter of the corpus's closing paragraphs as signature: precision was
308
+ // 24 percent and it started collapsing real content.
309
+ const strongPhrase = phraseStrength >= 0.9;
310
+ // A signature signs something. If the candidate run starts at the first authored block there is
311
+ // nothing above it to sign, and the message is a short note that happens to end in contact
312
+ // details, not a signature block. Without this guard the region swallowed whole short messages
313
+ // and left only a collapsed footer visible, which the corpus checks caught on 42 of 3,000
314
+ // messages.
315
+ let hasContentAbove = false;
316
+ for (let i = 0; i < signatureStart; i++) {
317
+ const block = blocks[i];
318
+ const decision = decisions[i];
319
+ if (block && decision && !block.features.isAllBlank && decision.label === 'NEW_CONTENT' && hasWords(block)) {
320
+ hasContentAbove = true;
321
+ break;
322
+ }
323
+ }
324
+ // The sender's own name counts alongside contact details and sign-off phrases. Requiring two
325
+ // signals still holds, so a message that merely mentions the sender's name in passing does not
326
+ // become a signature.
327
+ const signals = [hasContact, hasPhrase, hasSenderName].filter(Boolean).length;
328
+ const accepted = hasContentAbove && (strongPhrase || signals >= 2);
329
+ if (signatureStart !== -1 && accepted) {
330
+ const confidence = phraseStrength >= 0.9 ? 0.95 : signals >= 3 ? 0.9 : signals >= 2 ? 0.85 : 0.76;
331
+ const reason = phraseStrength >= 0.9 ? 'mobile_signature_phrase' : 'signature_position_and_length';
332
+ for (let i = signatureStart; i < authoredEnd; i++) {
333
+ const decision = decisions[i];
334
+ const block = blocks[i];
335
+ if (!decision || !block || decision.label === 'DISCLAIMER') {
336
+ continue;
337
+ }
338
+ if (isQuote(block) || isAnchorBlock(block)) {
339
+ break;
340
+ }
341
+ set(i, 'SIGNATURE', confidence, [reason]);
342
+ }
343
+ }
344
+ // Pass 3: regions.
345
+ const runs = findRuns(blocks);
346
+ const quotedRuns = runs.filter(run => run.quoted);
347
+ const contentRunsAfterFirstQuote = runs.filter(run => !run.quoted && quotedRuns.length > 0 && run.start > quotedRuns[0].start);
348
+ // Interleaved: more than one quoted run, with authored content between them. Every quoted run
349
+ // is then context for the answer beside it, not disposable history.
350
+ const interleaved = quotedRuns.length > 1 &&
351
+ contentRunsAfterFirstQuote.some(run => {
352
+ for (let i = run.start; i <= run.end; i++) {
353
+ const decision = decisions[i];
354
+ const block = blocks[i];
355
+ if (!decision || !block) {
356
+ continue;
357
+ }
358
+ if (decision.label === 'NEW_CONTENT' && hasWords(block) && !isAnchorBlock(block)) {
359
+ return true;
360
+ }
361
+ }
362
+ return false;
363
+ });
364
+ if (forwardAnchor !== -1) {
365
+ // Everything from the forward marker to the end is forwarded material. It is collapsed and
366
+ // kept, never hidden: the sender included it deliberately.
367
+ for (let i = forwardAnchor; i < blocks.length; i++) {
368
+ const decision = decisions[i];
369
+ if (!decision || decision.label === 'DISCLAIMER') {
370
+ continue;
371
+ }
372
+ set(i, 'FORWARDED_CONTENT', Math.min(0.95, forwardAnchorStrength), ['preceded_by_forward_header']);
373
+ }
374
+ }
375
+ for (const run of quotedRuns) {
376
+ if (forwardAnchor !== -1 && run.start >= forwardAnchor) {
377
+ continue;
378
+ }
379
+ const anchored = replyAnchor !== -1 && replyAnchor <= run.end;
380
+ const anchorStrength = anchored ? replyAnchorStrength : 0;
381
+ // Is there authored content after this run that is not a footer?
382
+ let newContentAfter = false;
383
+ for (let i = run.end + 1; i < blocks.length; i++) {
384
+ const decision = decisions[i];
385
+ const block = blocks[i];
386
+ if (!decision || !block || block.features.isAllBlank) {
387
+ continue;
388
+ }
389
+ if (decision.label === 'SIGNATURE' || decision.label === 'DISCLAIMER') {
390
+ continue;
391
+ }
392
+ if (isQuote(block) || isAnchorBlock(block)) {
393
+ continue;
394
+ }
395
+ if (hasWords(block)) {
396
+ newContentAfter = true;
397
+ break;
398
+ }
399
+ }
400
+ let label;
401
+ let confidence;
402
+ const reasons = [];
403
+ // Deepest quoting in this run. Depth 1 is the message being replied to; depth 2 or more is
404
+ // a quote inside a quote, which someone else already quoted before this conversation
405
+ // reached the sender.
406
+ let runDepth = 0;
407
+ for (let i = run.start; i <= run.end; i++) {
408
+ const block = blocks[i];
409
+ if (block) {
410
+ runDepth = Math.max(runDepth, block.features.quoteDepth, block.features.blockquoteDepth);
411
+ }
412
+ }
413
+ // Inline quoting is answering someone point by point, and it happens at depth 1: you quote
414
+ // the person you are replying to and write underneath. Nested quoting is a chain, and a run
415
+ // with an answer somewhere after it elsewhere in the message says nothing about whether this
416
+ // particular nested run is context. Measured on the reference set, 220 of the blocks wrongly
417
+ // called inline quote were at depth 2 or deeper.
418
+ const eligibleForInline = runDepth <= 1;
419
+ if (interleaved && eligibleForInline && newContentAfter) {
420
+ // Answered here, so it is the context for that answer.
421
+ label = 'INLINE_QUOTE';
422
+ confidence = 0.9;
423
+ reasons.push('interleaved_with_new_content', 'new_content_after_region');
424
+ }
425
+ else if (anchored) {
426
+ label = 'REPLY_HISTORY';
427
+ if (anchored) {
428
+ reasons.push(replyAnchor === containerAnchor ? 'inside_quote_container' : 'preceded_by_reply_header');
429
+ }
430
+ if (newContentAfter) {
431
+ // Bottom posting. The quote is still history, but the reader is depending on it for
432
+ // context more often than in the top posted case, so it is collapsed rather than
433
+ // hidden unless history matching confirmed it.
434
+ reasons.push('new_content_after_region');
435
+ confidence = 0.9;
436
+ }
437
+ else {
438
+ reasons.push('final_contiguous_quote_region', 'no_new_content_after_region');
439
+ confidence = Math.min(0.97, 0.6 + anchorStrength * 0.4);
440
+ }
441
+ }
442
+ else if (newContentAfter && eligibleForInline) {
443
+ // Quoted text with an answer under it and nothing saying it is history. This is the
444
+ // inline question and answer shape, and the quote is the context for the answer.
445
+ label = 'INLINE_QUOTE';
446
+ confidence = 0.9;
447
+ reasons.push('new_content_after_region', 'no_anchor_for_quote');
448
+ }
449
+ else {
450
+ // Quoted text at the end of a message with no anchor at all. It looks like history but
451
+ // nothing confirms it, so it stays visible.
452
+ label = 'UNKNOWN';
453
+ confidence = 0.5;
454
+ reasons.push('no_anchor_for_quote', 'ambiguous_kept_visible');
455
+ }
456
+ for (let i = run.start; i <= run.end; i++) {
457
+ const decision = decisions[i];
458
+ const block = blocks[i];
459
+ if (!decision || !block) {
460
+ continue;
461
+ }
462
+ if (decision.label === 'SIGNATURE' || decision.label === 'DISCLAIMER') {
463
+ continue;
464
+ }
465
+ // A generic blockquote alone is weaker evidence than a client specific container.
466
+ const weakContainer = block.evidence.some(item => item.id === 'html:generic_blockquote') && strengthOf(block, ['quote-container']) < config.minAnchorStrength;
467
+ const adjusted = label === 'REPLY_HISTORY' && weakContainer && !anchored ? Math.min(confidence, 0.8) : confidence;
468
+ set(i, label, adjusted, [...reasons]);
469
+ }
470
+ }
471
+ // Unmarked history after a strong anchor.
472
+ //
473
+ // Outlook and several other clients embed the previous message as plain paragraphs with no
474
+ // quote prefix at all, introduced by "-----Original Message-----" and a From/Sent/To/Subject
475
+ // block. Nothing in that text is marked as quoted, so the run rules above never see it and the
476
+ // whole previous message stays visible as new content.
477
+ //
478
+ // The sweep is bounded deliberately: it runs from the anchor only as far as the first quoted
479
+ // block. That is what keeps a bottom posted reply safe. In "On X wrote: / > quote / my answer",
480
+ // the anchor is immediately followed by a quoted block, so the sweep covers the anchor alone
481
+ // and the answer below is untouched.
482
+ if (replyAnchor !== -1 && replyAnchorStrength >= config.minAnchorStrength && !interleaved) {
483
+ let limit = blocks.length;
484
+ for (let i = replyAnchor + 1; i < blocks.length; i++) {
485
+ const block = blocks[i];
486
+ if (isQuote(block)) {
487
+ limit = i;
488
+ break;
489
+ }
490
+ }
491
+ // Corroboration: Outlook's shape is the separator followed closely by the embedded header
492
+ // block. With it, this is safe to hide. Without it, the region is collapsed but stays
493
+ // recoverable, because a lone anchor could in principle be text the sender typed.
494
+ let corroborated = false;
495
+ let seen = 0;
496
+ for (let i = replyAnchor; i < blocks.length && seen < 4; i++) {
497
+ const block = blocks[i];
498
+ if (block.features.isAllBlank) {
499
+ continue;
500
+ }
501
+ seen++;
502
+ if (block.kind === 'quoted_header_block' || block.features.headerFieldLines >= 2) {
503
+ corroborated = true;
504
+ break;
505
+ }
506
+ }
507
+ const reasons = corroborated
508
+ ? ['preceded_by_reply_header', 'contains_old_message_headers', 'no_new_content_after_region']
509
+ : ['preceded_by_reply_header', 'ambiguous_kept_visible'];
510
+ const confidence = corroborated ? 0.96 : 0.9;
511
+ for (let i = replyAnchor; i < limit; i++) {
512
+ const decision = decisions[i];
513
+ if (!decision) {
514
+ continue;
515
+ }
516
+ if (decision.label === 'FORWARDED_CONTENT' || decision.label === 'INLINE_QUOTE') {
517
+ continue;
518
+ }
519
+ // The attribution line was already classified by the run rules, which could not see the
520
+ // unmarked history that follows it. Raising it to the sweep's confidence keeps the whole
521
+ // region as one segment instead of splitting it at the anchor.
522
+ if (decision.label === 'REPLY_HISTORY' && decision.confidence >= confidence) {
523
+ continue;
524
+ }
525
+ decision.label = 'REPLY_HISTORY';
526
+ decision.confidence = confidence;
527
+ decision.reasons = [...reasons];
528
+ }
529
+ }
530
+ // A client reply separator makes everything after it history, whether or not it is quoted.
531
+ // Footers are left alone: a signature after the separator belongs to the quoted message and is
532
+ // already inside the region, while one before it belongs to the sender.
533
+ if (separatorAnchor !== -1 && !interleaved) {
534
+ const reasons = ['outlook_reply_separator', 'final_contiguous_quote_region'];
535
+ for (let i = separatorAnchor; i < blocks.length; i++) {
536
+ const decision = decisions[i];
537
+ const block = blocks[i];
538
+ if (!decision || !block) {
539
+ continue;
540
+ }
541
+ if (decision.label === 'FORWARDED_CONTENT' || decision.label === 'REPLY_HISTORY' || decision.label === 'INLINE_QUOTE') {
542
+ continue;
543
+ }
544
+ decision.label = 'REPLY_HISTORY';
545
+ decision.confidence = Math.min(0.96, 0.6 + separatorStrength * 0.4);
546
+ decision.reasons = [...reasons];
547
+ }
548
+ }
549
+ // An anchor line that sits outside any quoted run still belongs to the region it introduces.
550
+ for (const block of blocks) {
551
+ if (!isAnchorBlock(block)) {
552
+ continue;
553
+ }
554
+ const decision = decisions[block.index];
555
+ if (!decision || decision.label !== 'NEW_CONTENT') {
556
+ continue;
557
+ }
558
+ const next = decisions[block.index + 1] ?? decisions[block.index + 2];
559
+ if (next && (next.label === 'REPLY_HISTORY' || next.label === 'INLINE_QUOTE' || next.label === 'FORWARDED_CONTENT')) {
560
+ set(block.index, next.label, next.confidence, [...next.reasons]);
561
+ }
562
+ }
563
+ // Stage 5 evidence, applied strictly per block.
564
+ //
565
+ // A match proves provenance: this exact text existed before the message was written. It does
566
+ // not prove function. So a match may confirm a region the structure already read as history,
567
+ // and may rescue a quote that had no anchor at all, but it may never override a functional
568
+ // judgment:
569
+ //
570
+ // - INLINE_QUOTE is never promoted. Every inline quote appears in the parent by definition,
571
+ // so matching would hide exactly the context an answer depends on.
572
+ // - NEW_CONTENT is never promoted. A sender restating something from the parent is still
573
+ // writing it now.
574
+ //
575
+ // Applying this per block rather than per run matters. An earlier version promoted a whole run
576
+ // when any block in it matched, which hid authored text sitting beside quoted text and produced
577
+ // the only catastrophic failures measured on the reference set.
578
+ for (const index of matched) {
579
+ const decision = decisions[index];
580
+ const block = blocks[index];
581
+ if (!decision || !block) {
582
+ continue;
583
+ }
584
+ if (decision.label === 'REPLY_HISTORY') {
585
+ // Provenance alone does not license hiding. A quoted block with the sender's answer
586
+ // underneath is still the context for that answer, and a forwarded message is still
587
+ // something the sender chose to pass on, however certainly it existed before. Both stay
588
+ // collapsed, so a match confirms without ever crossing the hide threshold on its own.
589
+ const trailingRegion = decision.reasons.includes('no_new_content_after_region');
590
+ const insideForward = forwardAnchor !== -1 && index >= forwardAnchor;
591
+ const ceiling = trailingRegion && !insideForward ? 0.98 : 0.94;
592
+ decision.confidence = Math.max(decision.confidence, ceiling);
593
+ if (!decision.reasons.includes('matched_parent_message')) {
594
+ decision.reasons.push('matched_parent_message');
595
+ }
596
+ continue;
597
+ }
598
+ if (decision.label === 'UNKNOWN' && (isQuote(block) || isAnchorBlock(block))) {
599
+ decision.label = 'REPLY_HISTORY';
600
+ decision.confidence = 0.96;
601
+ decision.reasons = ['matched_parent_message', 'quote_depth_positive'];
602
+ }
603
+ }
604
+ // Pass 4: blank blocks carry no meaning of their own, so they inherit from their neighbour to
605
+ // avoid gaps in a hidden region.
606
+ for (let i = 0; i < decisions.length; i++) {
607
+ const decision = decisions[i];
608
+ const block = blocks[i];
609
+ if (!decision || !block || !block.features.isAllBlank) {
610
+ continue;
611
+ }
612
+ const previous = decisions[i - 1];
613
+ const next = decisions[i + 1];
614
+ if (previous && next && previous.label === next.label) {
615
+ decision.label = previous.label;
616
+ decision.confidence = Math.min(previous.confidence, next.confidence);
617
+ decision.reasons = [...previous.reasons];
618
+ }
619
+ else if (previous) {
620
+ decision.label = previous.label;
621
+ decision.confidence = previous.confidence;
622
+ decision.reasons = [...previous.reasons];
623
+ }
624
+ }
625
+ // Final guard: a message must never come out as nothing but its own footer.
626
+ //
627
+ // The signature region can legitimately grow to cover everything a sender typed when the
628
+ // message is short. That leaves a reader looking at a collapsed footer and no message, so if no
629
+ // authored block survives, the footer is given back. Being wrong about a signature costs
630
+ // tidiness; being wrong this way costs the whole message.
631
+ const hasVisibleAuthored = decisions.some((decision, index) => {
632
+ const block = blocks[index];
633
+ return block && hasWords(block) && (decision.label === 'NEW_CONTENT' || decision.label === 'INLINE_QUOTE' || decision.label === 'UNKNOWN');
634
+ });
635
+ if (!hasVisibleAuthored) {
636
+ for (let i = 0; i < decisions.length; i++) {
637
+ const decision = decisions[i];
638
+ const block = blocks[i];
639
+ if (decision && block && decision.label === 'SIGNATURE' && hasWords(block)) {
640
+ decision.label = 'NEW_CONTENT';
641
+ decision.confidence = 0.7;
642
+ decision.reasons = ['ambiguous_kept_visible'];
643
+ }
644
+ }
645
+ }
646
+ return decisions;
647
+ }
648
+ //# sourceMappingURL=rules.js.map