@postalsys/email-content 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +288 -0
  3. package/dist/.tsbuildinfo +1 -0
  4. package/dist/analyze.d.ts +91 -0
  5. package/dist/analyze.d.ts.map +1 -0
  6. package/dist/analyze.js +274 -0
  7. package/dist/analyze.js.map +1 -0
  8. package/dist/annotate/client.d.ts +64 -0
  9. package/dist/annotate/client.d.ts.map +1 -0
  10. package/dist/annotate/client.js +139 -0
  11. package/dist/annotate/client.js.map +1 -0
  12. package/dist/annotate/llm.d.ts +46 -0
  13. package/dist/annotate/llm.d.ts.map +1 -0
  14. package/dist/annotate/llm.js +107 -0
  15. package/dist/annotate/llm.js.map +1 -0
  16. package/dist/annotate/prompt.d.ts +38 -0
  17. package/dist/annotate/prompt.d.ts.map +1 -0
  18. package/dist/annotate/prompt.js +165 -0
  19. package/dist/annotate/prompt.js.map +1 -0
  20. package/dist/annotate/store.d.ts +43 -0
  21. package/dist/annotate/store.d.ts.map +1 -0
  22. package/dist/annotate/store.js +46 -0
  23. package/dist/annotate/store.js.map +1 -0
  24. package/dist/canonical/html.d.ts +33 -0
  25. package/dist/canonical/html.d.ts.map +1 -0
  26. package/dist/canonical/html.js +156 -0
  27. package/dist/canonical/html.js.map +1 -0
  28. package/dist/canonical/text.d.ts +75 -0
  29. package/dist/canonical/text.d.ts.map +1 -0
  30. package/dist/canonical/text.js +217 -0
  31. package/dist/canonical/text.js.map +1 -0
  32. package/dist/classify/confidence.d.ts +38 -0
  33. package/dist/classify/confidence.d.ts.map +1 -0
  34. package/dist/classify/confidence.js +59 -0
  35. package/dist/classify/confidence.js.map +1 -0
  36. package/dist/classify/reconcile.d.ts +52 -0
  37. package/dist/classify/reconcile.d.ts.map +1 -0
  38. package/dist/classify/reconcile.js +100 -0
  39. package/dist/classify/reconcile.js.map +1 -0
  40. package/dist/classify/rules.d.ts +51 -0
  41. package/dist/classify/rules.d.ts.map +1 -0
  42. package/dist/classify/rules.js +648 -0
  43. package/dist/classify/rules.js.map +1 -0
  44. package/dist/cli/annotate.d.ts +23 -0
  45. package/dist/cli/annotate.d.ts.map +1 -0
  46. package/dist/cli/annotate.js +328 -0
  47. package/dist/cli/annotate.js.map +1 -0
  48. package/dist/cli/args.d.ts +17 -0
  49. package/dist/cli/args.d.ts.map +1 -0
  50. package/dist/cli/args.js +72 -0
  51. package/dist/cli/args.js.map +1 -0
  52. package/dist/cli/benchmark.d.ts +13 -0
  53. package/dist/cli/benchmark.d.ts.map +1 -0
  54. package/dist/cli/benchmark.js +172 -0
  55. package/dist/cli/benchmark.js.map +1 -0
  56. package/dist/cli/corpus.d.ts +9 -0
  57. package/dist/cli/corpus.d.ts.map +1 -0
  58. package/dist/cli/corpus.js +152 -0
  59. package/dist/cli/corpus.js.map +1 -0
  60. package/dist/cli/disagreements.d.ts +14 -0
  61. package/dist/cli/disagreements.d.ts.map +1 -0
  62. package/dist/cli/disagreements.js +133 -0
  63. package/dist/cli/disagreements.js.map +1 -0
  64. package/dist/cli/evaluate.d.ts +15 -0
  65. package/dist/cli/evaluate.d.ts.map +1 -0
  66. package/dist/cli/evaluate.js +215 -0
  67. package/dist/cli/evaluate.js.map +1 -0
  68. package/dist/cli/extract.d.ts +10 -0
  69. package/dist/cli/extract.d.ts.map +1 -0
  70. package/dist/cli/extract.js +139 -0
  71. package/dist/cli/extract.js.map +1 -0
  72. package/dist/cli/main.d.ts +9 -0
  73. package/dist/cli/main.d.ts.map +1 -0
  74. package/dist/cli/main.js +147 -0
  75. package/dist/cli/main.js.map +1 -0
  76. package/dist/corpus/analyze.d.ts +66 -0
  77. package/dist/corpus/analyze.d.ts.map +1 -0
  78. package/dist/corpus/analyze.js +214 -0
  79. package/dist/corpus/analyze.js.map +1 -0
  80. package/dist/corpus/migrations/index.d.ts +14 -0
  81. package/dist/corpus/migrations/index.d.ts.map +1 -0
  82. package/dist/corpus/migrations/index.js +158 -0
  83. package/dist/corpus/migrations/index.js.map +1 -0
  84. package/dist/corpus/sample.d.ts +66 -0
  85. package/dist/corpus/sample.d.ts.map +1 -0
  86. package/dist/corpus/sample.js +98 -0
  87. package/dist/corpus/sample.js.map +1 -0
  88. package/dist/corpus/scanner.d.ts +79 -0
  89. package/dist/corpus/scanner.d.ts.map +1 -0
  90. package/dist/corpus/scanner.js +256 -0
  91. package/dist/corpus/scanner.js.map +1 -0
  92. package/dist/corpus/stats.d.ts +76 -0
  93. package/dist/corpus/stats.d.ts.map +1 -0
  94. package/dist/corpus/stats.js +240 -0
  95. package/dist/corpus/stats.js.map +1 -0
  96. package/dist/corpus/store.d.ts +67 -0
  97. package/dist/corpus/store.d.ts.map +1 -0
  98. package/dist/corpus/store.js +196 -0
  99. package/dist/corpus/store.js.map +1 -0
  100. package/dist/corpus/thread-index.d.ts +31 -0
  101. package/dist/corpus/thread-index.d.ts.map +1 -0
  102. package/dist/corpus/thread-index.js +182 -0
  103. package/dist/corpus/thread-index.js.map +1 -0
  104. package/dist/corpus/worker.d.ts +21 -0
  105. package/dist/corpus/worker.d.ts.map +1 -0
  106. package/dist/corpus/worker.js +74 -0
  107. package/dist/corpus/worker.js.map +1 -0
  108. package/dist/detect/client.d.ts +28 -0
  109. package/dist/detect/client.d.ts.map +1 -0
  110. package/dist/detect/client.js +121 -0
  111. package/dist/detect/client.js.map +1 -0
  112. package/dist/detect/disclaimer.d.ts +12 -0
  113. package/dist/detect/disclaimer.d.ts.map +1 -0
  114. package/dist/detect/disclaimer.js +28 -0
  115. package/dist/detect/disclaimer.js.map +1 -0
  116. package/dist/detect/forward-header.d.ts +10 -0
  117. package/dist/detect/forward-header.d.ts.map +1 -0
  118. package/dist/detect/forward-header.js +35 -0
  119. package/dist/detect/forward-header.js.map +1 -0
  120. package/dist/detect/patterns/disclaimers.json +113 -0
  121. package/dist/detect/patterns/forward-headers.json +40 -0
  122. package/dist/detect/patterns/html-clients.json +157 -0
  123. package/dist/detect/patterns/quoted-headers.json +41 -0
  124. package/dist/detect/patterns/reply-headers.json +139 -0
  125. package/dist/detect/patterns/signatures.json +58 -0
  126. package/dist/detect/patterns.d.ts +43 -0
  127. package/dist/detect/patterns.d.ts.map +1 -0
  128. package/dist/detect/patterns.js +125 -0
  129. package/dist/detect/patterns.js.map +1 -0
  130. package/dist/detect/reply-header.d.ts +44 -0
  131. package/dist/detect/reply-header.d.ts.map +1 -0
  132. package/dist/detect/reply-header.js +134 -0
  133. package/dist/detect/reply-header.js.map +1 -0
  134. package/dist/detect/signature.d.ts +28 -0
  135. package/dist/detect/signature.d.ts.map +1 -0
  136. package/dist/detect/signature.js +77 -0
  137. package/dist/detect/signature.js.map +1 -0
  138. package/dist/detect/traffic.d.ts +26 -0
  139. package/dist/detect/traffic.d.ts.map +1 -0
  140. package/dist/detect/traffic.js +98 -0
  141. package/dist/detect/traffic.js.map +1 -0
  142. package/dist/eval/metrics.d.ts +87 -0
  143. package/dist/eval/metrics.d.ts.map +1 -0
  144. package/dist/eval/metrics.js +185 -0
  145. package/dist/eval/metrics.js.map +1 -0
  146. package/dist/index.d.ts +54 -0
  147. package/dist/index.d.ts.map +1 -0
  148. package/dist/index.js +163 -0
  149. package/dist/index.js.map +1 -0
  150. package/dist/match/history.d.ts +62 -0
  151. package/dist/match/history.d.ts.map +1 -0
  152. package/dist/match/history.js +128 -0
  153. package/dist/match/history.js.map +1 -0
  154. package/dist/match/normalize.d.ts +33 -0
  155. package/dist/match/normalize.d.ts.map +1 -0
  156. package/dist/match/normalize.js +63 -0
  157. package/dist/match/normalize.js.map +1 -0
  158. package/dist/match/shingle.d.ts +44 -0
  159. package/dist/match/shingle.d.ts.map +1 -0
  160. package/dist/match/shingle.js +78 -0
  161. package/dist/match/shingle.js.map +1 -0
  162. package/dist/mime/charset.d.ts +33 -0
  163. package/dist/mime/charset.d.ts.map +1 -0
  164. package/dist/mime/charset.js +118 -0
  165. package/dist/mime/charset.js.map +1 -0
  166. package/dist/mime/headers.d.ts +32 -0
  167. package/dist/mime/headers.d.ts.map +1 -0
  168. package/dist/mime/headers.js +127 -0
  169. package/dist/mime/headers.js.map +1 -0
  170. package/dist/mime/model.d.ts +111 -0
  171. package/dist/mime/model.d.ts.map +1 -0
  172. package/dist/mime/model.js +17 -0
  173. package/dist/mime/model.js.map +1 -0
  174. package/dist/mime/parse.d.ts +27 -0
  175. package/dist/mime/parse.d.ts.map +1 -0
  176. package/dist/mime/parse.js +343 -0
  177. package/dist/mime/parse.js.map +1 -0
  178. package/dist/segment/features.d.ts +29 -0
  179. package/dist/segment/features.d.ts.map +1 -0
  180. package/dist/segment/features.js +102 -0
  181. package/dist/segment/features.js.map +1 -0
  182. package/dist/segment/html-blocks.d.ts +28 -0
  183. package/dist/segment/html-blocks.d.ts.map +1 -0
  184. package/dist/segment/html-blocks.js +596 -0
  185. package/dist/segment/html-blocks.js.map +1 -0
  186. package/dist/segment/text-blocks.d.ts +30 -0
  187. package/dist/segment/text-blocks.d.ts.map +1 -0
  188. package/dist/segment/text-blocks.js +204 -0
  189. package/dist/segment/text-blocks.js.map +1 -0
  190. package/dist/segment/types.d.ts +88 -0
  191. package/dist/segment/types.d.ts.map +1 -0
  192. package/dist/segment/types.js +11 -0
  193. package/dist/segment/types.js.map +1 -0
  194. package/dist/types.d.ts +85 -0
  195. package/dist/types.d.ts.map +1 -0
  196. package/dist/types.js +17 -0
  197. package/dist/types.js.map +1 -0
  198. package/dist/util/hash.d.ts +49 -0
  199. package/dist/util/hash.d.ts.map +1 -0
  200. package/dist/util/hash.js +0 -0
  201. package/dist/util/hash.js.map +1 -0
  202. package/dist/util/lang.d.ts +25 -0
  203. package/dist/util/lang.d.ts.map +1 -0
  204. package/dist/util/lang.js +135 -0
  205. package/dist/util/lang.js.map +1 -0
  206. package/dist/util/worker-pool.d.ts +51 -0
  207. package/dist/util/worker-pool.d.ts.map +1 -0
  208. package/dist/util/worker-pool.js +155 -0
  209. package/dist/util/worker-pool.js.map +1 -0
  210. package/package.json +74 -0
@@ -0,0 +1,128 @@
1
+ "use strict";
2
+ /**
3
+ * Stage 5: matching blocks against messages that already existed.
4
+ *
5
+ * This is the strongest evidence the pipeline can have. Every other signal is an inference from
6
+ * markup or wording: an attribution line suggests a quote follows, a blockquote suggests its
7
+ * contents are old. A match against the actual parent message is not an inference. The text
8
+ * demonstrably existed before this message was written, so it cannot be something the sender just
9
+ * wrote.
10
+ *
11
+ * In the working corpus 90.9 percent of replies resolve their In-Reply-To to a message that is also
12
+ * present, which makes this unusually available here. It remains strictly optional: extraction
13
+ * works with no history at all, and history only ever adds evidence.
14
+ *
15
+ * The matcher is deliberately conservative in one direction. A match can promote a block to
16
+ * REPLY_HISTORY, which hides it, so short blocks are never matched, and a high containment
17
+ * threshold is required. A non match never demotes anything.
18
+ */
19
+ Object.defineProperty(exports, "__esModule", { value: true });
20
+ exports.buildHistoryDocument = buildHistoryDocument;
21
+ exports.buildHistoryDocumentFromContent = buildHistoryDocumentFromContent;
22
+ exports.matchBlocks = matchBlocks;
23
+ exports.matchedIndices = matchedIndices;
24
+ const parse_1 = require("../mime/parse");
25
+ const text_1 = require("../canonical/text");
26
+ const html_1 = require("../canonical/html");
27
+ const shingle_1 = require("./shingle");
28
+ const normalize_1 = require("./normalize");
29
+ /**
30
+ * Build a searchable document from a raw earlier message.
31
+ *
32
+ * Both representations are folded into one text projection. Which representation the earlier
33
+ * message used has nothing to do with which one the reply quoted from, and clients routinely quote
34
+ * the HTML of a message into the plain text of a reply.
35
+ */
36
+ async function buildHistoryDocument(raw, id, isParent) {
37
+ const buffer = typeof raw === 'string' ? Buffer.from(raw) : raw;
38
+ const message = await (0, parse_1.parseMessage)(buffer);
39
+ const parts = [];
40
+ for (const part of message.parts) {
41
+ if (!part.decoded || part.disposition === 'attachment') {
42
+ continue;
43
+ }
44
+ if (part.contentType === 'text/plain') {
45
+ parts.push((0, text_1.canonicaliseText)(part.decoded.text).text);
46
+ }
47
+ else if (part.contentType === 'text/html') {
48
+ parts.push((0, html_1.stripTagsToText)(part.decoded.text));
49
+ }
50
+ }
51
+ const normalised = (0, normalize_1.normaliseForMatch)(parts.join('\n'));
52
+ return { id, normalised, index: (0, shingle_1.buildShingleIndex)(normalised), isParent };
53
+ }
54
+ /**
55
+ * Build a searchable document from already parsed content.
56
+ *
57
+ * The counterpart to buildHistoryDocument for callers that never had a raw message. Both
58
+ * representations are folded together for the same reason: which one the earlier message used has
59
+ * nothing to do with which one the reply quoted from.
60
+ */
61
+ function buildHistoryDocumentFromContent(text, html, id, isParent) {
62
+ const parts = [];
63
+ if (text) {
64
+ parts.push((0, text_1.canonicaliseText)(text).text);
65
+ }
66
+ if (html) {
67
+ parts.push((0, html_1.stripTagsToText)(html));
68
+ }
69
+ const normalised = (0, normalize_1.normaliseForMatch)(parts.join('\n'));
70
+ return { id, normalised, index: (0, shingle_1.buildShingleIndex)(normalised), isParent };
71
+ }
72
+ /**
73
+ * Match blocks against known history.
74
+ *
75
+ * Returns one match per block at most, preferring the direct parent over older ancestors, and
76
+ * higher containment over lower.
77
+ */
78
+ function matchBlocks(blocks, history) {
79
+ if (!history.length) {
80
+ return [];
81
+ }
82
+ const matches = [];
83
+ for (const block of blocks) {
84
+ const normalised = (0, normalize_1.normaliseForMatch)(block.content);
85
+ if (!normalised) {
86
+ continue;
87
+ }
88
+ const index = (0, shingle_1.buildShingleIndex)(normalised);
89
+ if (!(0, shingle_1.isMatchable)(index)) {
90
+ // Too short to judge. "Thanks" appearing in the parent proves nothing about this block.
91
+ continue;
92
+ }
93
+ let best = null;
94
+ for (const document of history) {
95
+ // Tier one: the normalised text appears verbatim in the earlier message.
96
+ const exact = document.normalised.includes(normalised);
97
+ const score = exact ? 1 : (0, shingle_1.containment)(index, document.index).containment;
98
+ if (score < shingle_1.CONTAINMENT_THRESHOLD) {
99
+ continue;
100
+ }
101
+ const candidate = {
102
+ blockIndex: block.index,
103
+ documentId: document.id,
104
+ isParent: document.isParent,
105
+ containment: score,
106
+ exact
107
+ };
108
+ if (!best || betterMatch(candidate, best)) {
109
+ best = candidate;
110
+ }
111
+ }
112
+ if (best) {
113
+ matches.push(best);
114
+ }
115
+ }
116
+ return matches;
117
+ }
118
+ function betterMatch(candidate, current) {
119
+ if (candidate.isParent !== current.isParent) {
120
+ return candidate.isParent;
121
+ }
122
+ return candidate.containment > current.containment;
123
+ }
124
+ /** Convenience view for the classifier, which only needs to know which indices matched. */
125
+ function matchedIndices(matches) {
126
+ return new Set(matches.map(match => match.blockIndex));
127
+ }
128
+ //# sourceMappingURL=history.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;AAmCH,oDAkBC;AASD,0EAUC;AAQD,kCA4CC;AAUD,wCAEC;AAtID,yCAA6C;AAC7C,4CAAqD;AACrD,4CAAoD;AAEpD,uCAAkH;AAClH,2CAAgD;AAqBhD;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,GAAoB,EAAE,EAAU,EAAE,QAAiB;IAC1F,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACrD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,+BAA+B,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU,EAAE,QAAiB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,MAAwB,EAAE,OAAmC;IACrF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,2BAAiB,EAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAA,qBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;YACtB,wFAAwF;YACxF,SAAS;QACb,CAAC;QAED,IAAI,IAAI,GAAsB,IAAI,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC7B,yEAAyE;YACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,qBAAW,EAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC;YACzE,IAAI,KAAK,GAAG,+BAAqB,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAe;gBAC1B,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,KAAK;gBAClB,KAAK;aACR,CAAC;YACF,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBACxC,IAAI,GAAG,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,SAAqB,EAAE,OAAmB;IAC3D,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,SAAS,CAAC,QAAQ,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;AACvD,CAAC;AAED,2FAA2F;AAC3F,SAAgB,cAAc,CAAC,OAA8B;IACzD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AAC3D,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Normalisation for history matching.
3
+ *
4
+ * The same sentence, quoted by a mail client, is rarely byte identical to the original. It has been
5
+ * re-wrapped, prefixed with ">", had its entities decoded or re-encoded, had non-breaking spaces
6
+ * substituted, had smart quotes swapped for straight ones, and may have been transcoded through a
7
+ * legacy charset and back.
8
+ *
9
+ * Matching therefore happens on a normalised projection: aggressive enough to survive all of that,
10
+ * conservative enough that two genuinely different sentences do not collide. Note the asymmetry in
11
+ * consequences here is the opposite of the rest of the pipeline: a false match causes content to be
12
+ * hidden, so this normalisation deliberately stops short of stemming, case folding of proper nouns,
13
+ * or anything else that would let distinct text collapse together.
14
+ */
15
+ /**
16
+ * Normalise text for comparison.
17
+ *
18
+ * Quote prefixes are stripped, whitespace is collapsed to single spaces, punctuation variants are
19
+ * folded, and the result is lowercased. Line structure is deliberately discarded, because re-wrapping
20
+ * is the single most common transformation a quoted message undergoes.
21
+ */
22
+ export declare function normaliseForMatch(text: string): string;
23
+ /** Word tokens used for shingling and alignment. */
24
+ export declare function tokenise(normalised: string): string[];
25
+ /**
26
+ * Minimum tokens before a block is worth matching at all.
27
+ *
28
+ * Short blocks match by coincidence. "Thanks", "Yes", "Best regards" and "OK" appear in almost
29
+ * every thread, so a match on them says nothing about whether this particular text was quoted. Nine
30
+ * tokens is roughly a full sentence.
31
+ */
32
+ export declare const MIN_MATCH_TOKENS = 9;
33
+ //# sourceMappingURL=normalize.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/match/normalize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAaH;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYtD;AAED,oDAAoD;AACpD,wBAAgB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAErD;AAED;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAC"}
@@ -0,0 +1,63 @@
1
+ "use strict";
2
+ /**
3
+ * Normalisation for history matching.
4
+ *
5
+ * The same sentence, quoted by a mail client, is rarely byte identical to the original. It has been
6
+ * re-wrapped, prefixed with ">", had its entities decoded or re-encoded, had non-breaking spaces
7
+ * substituted, had smart quotes swapped for straight ones, and may have been transcoded through a
8
+ * legacy charset and back.
9
+ *
10
+ * Matching therefore happens on a normalised projection: aggressive enough to survive all of that,
11
+ * conservative enough that two genuinely different sentences do not collide. Note the asymmetry in
12
+ * consequences here is the opposite of the rest of the pipeline: a false match causes content to be
13
+ * hidden, so this normalisation deliberately stops short of stemming, case folding of proper nouns,
14
+ * or anything else that would let distinct text collapse together.
15
+ */
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.MIN_MATCH_TOKENS = void 0;
18
+ exports.normaliseForMatch = normaliseForMatch;
19
+ exports.tokenise = tokenise;
20
+ /** Punctuation and whitespace variants that clients substitute freely. */
21
+ const REPLACEMENTS = [
22
+ [/[\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]/g, ' '], // exotic spaces
23
+ [/[\u200b-\u200d\ufeff\u00ad]/g, ''], // zero width and soft hyphens
24
+ [/[‘’‚‛′‵]/g, "'"], // single quotes
25
+ [/[“”„‟″‶]/g, '"'], // double quotes
26
+ [/[‐-―−]/g, '-'], // dashes
27
+ [/[…]/g, '...'],
28
+ [/[•·⁃]/g, '*'] // bullets
29
+ ];
30
+ /**
31
+ * Normalise text for comparison.
32
+ *
33
+ * Quote prefixes are stripped, whitespace is collapsed to single spaces, punctuation variants are
34
+ * folded, and the result is lowercased. Line structure is deliberately discarded, because re-wrapping
35
+ * is the single most common transformation a quoted message undergoes.
36
+ */
37
+ function normaliseForMatch(text) {
38
+ let out = text;
39
+ try {
40
+ out = out.normalize('NFC');
41
+ }
42
+ catch {
43
+ // Unnormalisable input still compares usefully.
44
+ }
45
+ out = out.replace(/^[ \t]{0,3}(?:>[ \t]?)+/gm, '');
46
+ for (const [pattern, replacement] of REPLACEMENTS) {
47
+ out = out.replace(pattern, replacement);
48
+ }
49
+ return out.replace(/\s+/g, ' ').trim().toLowerCase();
50
+ }
51
+ /** Word tokens used for shingling and alignment. */
52
+ function tokenise(normalised) {
53
+ return normalised.split(/[^\p{L}\p{N}]+/u).filter(token => token.length > 0);
54
+ }
55
+ /**
56
+ * Minimum tokens before a block is worth matching at all.
57
+ *
58
+ * Short blocks match by coincidence. "Thanks", "Yes", "Best regards" and "OK" appear in almost
59
+ * every thread, so a match on them says nothing about whether this particular text was quoted. Nine
60
+ * tokens is roughly a full sentence.
61
+ */
62
+ exports.MIN_MATCH_TOKENS = 9;
63
+ //# sourceMappingURL=normalize.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/match/normalize.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;AAoBH,8CAYC;AAGD,4BAEC;AAnCD,0EAA0E;AAC1E,MAAM,YAAY,GAAgC;IAC9C,CAAC,gDAAgD,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACzE,CAAC,8BAA8B,EAAE,EAAE,CAAC,EAAE,8BAA8B;IACpE,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACpC,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACpC,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,SAAS;IAC3B,CAAC,MAAM,EAAE,KAAK,CAAC;IACf,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU;CAC7B,CAAC;AAEF;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,IAAY;IAC1C,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,IAAI,CAAC;QACD,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACL,gDAAgD;IACpD,CAAC;IACD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC;QAChD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACzD,CAAC;AAED,oDAAoD;AACpD,SAAgB,QAAQ,CAAC,UAAkB;IACvC,OAAO,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjF,CAAC;AAED;;;;;;GAMG;AACU,QAAA,gBAAgB,GAAG,CAAC,CAAC"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Token shingle matching with rolling hashes.
3
+ *
4
+ * Stage 5's middle tier. Exact normalised comparison catches text that was quoted verbatim;
5
+ * shingles catch text that was quoted with an edit, a truncation, or an interleaved insertion,
6
+ * which clients and people both do.
7
+ *
8
+ * A shingle is a window of consecutive tokens. Two texts sharing most of their shingles share most
9
+ * of their content regardless of where it moved to. The hash is rolling, so building the index for
10
+ * a parent message costs one pass over its tokens.
11
+ */
12
+ /** Tokens per shingle. Long enough to be specific, short enough to survive small edits. */
13
+ export declare const SHINGLE_SIZE = 5;
14
+ export interface ShingleIndex {
15
+ hashes: Set<number>;
16
+ tokenCount: number;
17
+ }
18
+ /**
19
+ * Build a shingle index over normalised text.
20
+ *
21
+ * The rolling hash is a simple polynomial over per token 32 bit hashes. Collisions are possible and
22
+ * harmless at this scale: a collision inflates a similarity score slightly, and the containment
23
+ * threshold plus the corroboration rules downstream absorb that.
24
+ */
25
+ export declare function buildShingleIndex(normalised: string): ShingleIndex;
26
+ export interface ContainmentResult {
27
+ /** Share of the block's shingles that also appear in the reference. */
28
+ containment: number;
29
+ /** Shingles compared. Zero means the block was too short to judge. */
30
+ shingles: number;
31
+ }
32
+ /**
33
+ * How much of `block` is contained in `reference`.
34
+ *
35
+ * Containment rather than Jaccard similarity, deliberately: a quoted paragraph is a small part of a
36
+ * large parent message, so symmetric similarity would be near zero for a perfect quote. What
37
+ * matters is whether everything in this block already existed.
38
+ */
39
+ export declare function containment(block: ShingleIndex, reference: ShingleIndex): ContainmentResult;
40
+ /** Containment at or above this counts as a match, once the block is long enough to judge. */
41
+ export declare const CONTAINMENT_THRESHOLD = 0.8;
42
+ /** True when a block is long enough for a containment score to mean anything. */
43
+ export declare function isMatchable(index: ShingleIndex): boolean;
44
+ //# sourceMappingURL=shingle.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shingle.d.ts","sourceRoot":"","sources":["../../src/match/shingle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,2FAA2F;AAC3F,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,CAclE;AAYD,MAAM,WAAW,iBAAiB;IAC9B,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,GAAG,iBAAiB,CAW3F;AAED,8FAA8F;AAC9F,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAExD"}
@@ -0,0 +1,78 @@
1
+ "use strict";
2
+ /**
3
+ * Token shingle matching with rolling hashes.
4
+ *
5
+ * Stage 5's middle tier. Exact normalised comparison catches text that was quoted verbatim;
6
+ * shingles catch text that was quoted with an edit, a truncation, or an interleaved insertion,
7
+ * which clients and people both do.
8
+ *
9
+ * A shingle is a window of consecutive tokens. Two texts sharing most of their shingles share most
10
+ * of their content regardless of where it moved to. The hash is rolling, so building the index for
11
+ * a parent message costs one pass over its tokens.
12
+ */
13
+ Object.defineProperty(exports, "__esModule", { value: true });
14
+ exports.CONTAINMENT_THRESHOLD = exports.SHINGLE_SIZE = void 0;
15
+ exports.buildShingleIndex = buildShingleIndex;
16
+ exports.containment = containment;
17
+ exports.isMatchable = isMatchable;
18
+ const hash_1 = require("../util/hash");
19
+ const normalize_1 = require("./normalize");
20
+ /** Tokens per shingle. Long enough to be specific, short enough to survive small edits. */
21
+ exports.SHINGLE_SIZE = 5;
22
+ /**
23
+ * Build a shingle index over normalised text.
24
+ *
25
+ * The rolling hash is a simple polynomial over per token 32 bit hashes. Collisions are possible and
26
+ * harmless at this scale: a collision inflates a similarity score slightly, and the containment
27
+ * threshold plus the corroboration rules downstream absorb that.
28
+ */
29
+ function buildShingleIndex(normalised) {
30
+ const tokens = (0, normalize_1.tokenise)(normalised);
31
+ const hashes = new Set();
32
+ if (tokens.length < exports.SHINGLE_SIZE) {
33
+ // Short text is indexed as a single shingle so it can still match exactly.
34
+ if (tokens.length > 0) {
35
+ hashes.add(hashTokens(tokens, 0, tokens.length));
36
+ }
37
+ return { hashes, tokenCount: tokens.length };
38
+ }
39
+ for (let i = 0; i + exports.SHINGLE_SIZE <= tokens.length; i++) {
40
+ hashes.add(hashTokens(tokens, i, exports.SHINGLE_SIZE));
41
+ }
42
+ return { hashes, tokenCount: tokens.length };
43
+ }
44
+ function hashTokens(tokens, start, length) {
45
+ let hash = 0x811c9dc5;
46
+ for (let i = start; i < start + length; i++) {
47
+ const token = tokens[i];
48
+ hash = (Math.imul(hash, 31) + (0, hash_1.fnv1a32)(token)) >>> 0;
49
+ hash = (hash ^ (0, hash_1.fnv1a32b)(token)) >>> 0;
50
+ }
51
+ return hash >>> 0;
52
+ }
53
+ /**
54
+ * How much of `block` is contained in `reference`.
55
+ *
56
+ * Containment rather than Jaccard similarity, deliberately: a quoted paragraph is a small part of a
57
+ * large parent message, so symmetric similarity would be near zero for a perfect quote. What
58
+ * matters is whether everything in this block already existed.
59
+ */
60
+ function containment(block, reference) {
61
+ if (block.hashes.size === 0) {
62
+ return { containment: 0, shingles: 0 };
63
+ }
64
+ let hits = 0;
65
+ for (const hash of block.hashes) {
66
+ if (reference.hashes.has(hash)) {
67
+ hits++;
68
+ }
69
+ }
70
+ return { containment: hits / block.hashes.size, shingles: block.hashes.size };
71
+ }
72
+ /** Containment at or above this counts as a match, once the block is long enough to judge. */
73
+ exports.CONTAINMENT_THRESHOLD = 0.8;
74
+ /** True when a block is long enough for a containment score to mean anything. */
75
+ function isMatchable(index) {
76
+ return index.tokenCount >= normalize_1.MIN_MATCH_TOKENS;
77
+ }
78
+ //# sourceMappingURL=shingle.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shingle.js","sourceRoot":"","sources":["../../src/match/shingle.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAoBH,8CAcC;AA0BD,kCAWC;AAMD,kCAEC;AA7ED,uCAAiD;AACjD,2CAAyD;AAEzD,2FAA2F;AAC9E,QAAA,YAAY,GAAG,CAAC,CAAC;AAO9B;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,UAAkB;IAChD,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,UAAU,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,GAAG,oBAAY,EAAE,CAAC;QAC/B,2EAA2E;QAC3E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAY,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,oBAAY,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,KAAa,EAAE,MAAc;IACxE,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,KAAK,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;QAClC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAA,cAAO,EAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,GAAG,CAAC,IAAI,GAAG,IAAA,eAAQ,EAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACtB,CAAC;AASD;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,KAAmB,EAAE,SAAuB;IACpE,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,IAAI,EAAE,CAAC;QACX,CAAC;IACL,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClF,CAAC;AAED,8FAA8F;AACjF,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC,iFAAiF;AACjF,SAAgB,WAAW,CAAC,KAAmB;IAC3C,OAAO,KAAK,CAAC,UAAU,IAAI,4BAAgB,CAAC;AAChD,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Charset decoding with an explicit, recorded fallback chain.
3
+ *
4
+ * Measured on the working corpus: about 14 percent of text parts declare something other than
5
+ * utf-8, including iso-8859-1, iso-8859-15, windows-1257, windows-1252 and iso-2022-jp. A silent
6
+ * fallback would corrupt those messages without anyone noticing, so every downgrade is reported
7
+ * back to the caller.
8
+ *
9
+ * Chain:
10
+ * 1. Normalise the declared label through libmime (handles aliases such as "unicode-1-1-utf-8").
11
+ * 2. Node's built in TextDecoder, which covers the WHATWG encoding set including the legacy
12
+ * single byte and CJK encodings.
13
+ * 3. libmime's iconv-lite backed decoder, which covers labels TextDecoder rejects (utf-7 and a
14
+ * long tail of aliases).
15
+ * 4. windows-1252, which cannot fail, flagged as a downgrade.
16
+ */
17
+ export interface CharsetDecodeResult {
18
+ text: string;
19
+ charsetUsed: string;
20
+ replacementChars: number;
21
+ /** True when the declared charset could not be honoured. */
22
+ fellBack: boolean;
23
+ }
24
+ /** Normalise a raw charset label. Returns null when the label is empty or meaningless. */
25
+ export declare function normaliseCharset(label: string | null | undefined): string | null;
26
+ /**
27
+ * Decode a byte buffer using the declared charset, falling back predictably.
28
+ *
29
+ * @param buf raw bytes, already transfer decoded
30
+ * @param declared charset label from the Content-Type header, or null when absent
31
+ */
32
+ export declare function decodeCharset(buf: Buffer, declared: string | null): CharsetDecodeResult;
33
+ //# sourceMappingURL=charset.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"charset.d.ts","sourceRoot":"","sources":["../../src/mime/charset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAaH,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,0FAA0F;AAC1F,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAgBhF;AA6BD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAyCvF"}
@@ -0,0 +1,118 @@
1
+ "use strict";
2
+ /**
3
+ * Charset decoding with an explicit, recorded fallback chain.
4
+ *
5
+ * Measured on the working corpus: about 14 percent of text parts declare something other than
6
+ * utf-8, including iso-8859-1, iso-8859-15, windows-1257, windows-1252 and iso-2022-jp. A silent
7
+ * fallback would corrupt those messages without anyone noticing, so every downgrade is reported
8
+ * back to the caller.
9
+ *
10
+ * Chain:
11
+ * 1. Normalise the declared label through libmime (handles aliases such as "unicode-1-1-utf-8").
12
+ * 2. Node's built in TextDecoder, which covers the WHATWG encoding set including the legacy
13
+ * single byte and CJK encodings.
14
+ * 3. libmime's iconv-lite backed decoder, which covers labels TextDecoder rejects (utf-7 and a
15
+ * long tail of aliases).
16
+ * 4. windows-1252, which cannot fail, flagged as a downgrade.
17
+ */
18
+ var __importDefault = (this && this.__importDefault) || function (mod) {
19
+ return (mod && mod.__esModule) ? mod : { "default": mod };
20
+ };
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.normaliseCharset = normaliseCharset;
23
+ exports.decodeCharset = decodeCharset;
24
+ const charset_1 = __importDefault(require("libmime/lib/charset"));
25
+ const REPLACEMENT = '�';
26
+ /**
27
+ * A utf-8 decode that produces more than this share of replacement characters is treated as a
28
+ * mislabelled single byte encoding rather than as damaged utf-8. Chosen so that a single bad byte
29
+ * in a long message does not trigger a switch, while genuinely mislabelled Latin-1 text does.
30
+ */
31
+ const MOJIBAKE_RATIO = 0.02;
32
+ /** Normalise a raw charset label. Returns null when the label is empty or meaningless. */
33
+ function normaliseCharset(label) {
34
+ if (!label) {
35
+ return null;
36
+ }
37
+ const trimmed = label
38
+ .trim()
39
+ .replace(/^["']|["']$/g, '')
40
+ .toLowerCase();
41
+ if (!trimmed || trimmed === 'unknown' || trimmed === 'x-unknown' || trimmed === 'none') {
42
+ return null;
43
+ }
44
+ try {
45
+ return String(charset_1.default.normalizeCharset(trimmed) || trimmed).toLowerCase();
46
+ }
47
+ catch {
48
+ return trimmed;
49
+ }
50
+ }
51
+ function countReplacements(text) {
52
+ let count = 0;
53
+ let index = text.indexOf(REPLACEMENT);
54
+ while (index !== -1) {
55
+ count++;
56
+ index = text.indexOf(REPLACEMENT, index + 1);
57
+ }
58
+ return count;
59
+ }
60
+ function tryTextDecoder(buf, label) {
61
+ try {
62
+ return new TextDecoder(label, { fatal: false }).decode(buf);
63
+ }
64
+ catch {
65
+ return null;
66
+ }
67
+ }
68
+ function tryIconv(buf, label) {
69
+ try {
70
+ const decoded = charset_1.default.decode(buf, label);
71
+ return typeof decoded === 'string' ? decoded : null;
72
+ }
73
+ catch {
74
+ return null;
75
+ }
76
+ }
77
+ /**
78
+ * Decode a byte buffer using the declared charset, falling back predictably.
79
+ *
80
+ * @param buf raw bytes, already transfer decoded
81
+ * @param declared charset label from the Content-Type header, or null when absent
82
+ */
83
+ function decodeCharset(buf, declared) {
84
+ if (buf.length === 0) {
85
+ return { text: '', charsetUsed: normaliseCharset(declared) || 'utf-8', replacementChars: 0, fellBack: false };
86
+ }
87
+ const normalised = normaliseCharset(declared);
88
+ // RFC 2045 says us-ascii when absent. Decoding as utf-8 is a strict superset for valid ascii
89
+ // and recovers undeclared utf-8, which is far more common in practice than strict ascii.
90
+ const label = normalised || 'utf-8';
91
+ let text = tryTextDecoder(buf, label);
92
+ let charsetUsed = label;
93
+ let fellBack = false;
94
+ if (text === null) {
95
+ text = tryIconv(buf, label);
96
+ fellBack = text !== null;
97
+ }
98
+ if (text === null) {
99
+ text = tryTextDecoder(buf, 'windows-1252') ?? buf.toString('latin1');
100
+ charsetUsed = 'windows-1252';
101
+ fellBack = true;
102
+ }
103
+ let replacementChars = countReplacements(text);
104
+ // Mislabelled single byte content declared as utf-8 is common. Only switch when the damage is
105
+ // widespread, so that one stray byte in an otherwise valid utf-8 message is left alone.
106
+ const looksLikeUtf8Label = label === 'utf-8' || label === 'utf8' || !normalised;
107
+ if (replacementChars > 0 && looksLikeUtf8Label && replacementChars / Math.max(text.length, 1) > MOJIBAKE_RATIO) {
108
+ const retry = tryTextDecoder(buf, 'windows-1252');
109
+ if (retry !== null) {
110
+ text = retry;
111
+ charsetUsed = 'windows-1252';
112
+ fellBack = true;
113
+ replacementChars = countReplacements(text);
114
+ }
115
+ }
116
+ return { text, charsetUsed, replacementChars, fellBack };
117
+ }
118
+ //# sourceMappingURL=charset.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"charset.js","sourceRoot":"","sources":["../../src/mime/charset.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;;;AAsBH,4CAgBC;AAmCD,sCAyCC;AAhHD,kEAA6C;AAE7C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB;;;;GAIG;AACH,MAAM,cAAc,GAAG,IAAI,CAAC;AAU5B,0FAA0F;AAC1F,SAAgB,gBAAgB,CAAC,KAAgC;IAC7D,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,KAAK;SAChB,IAAI,EAAE;SACN,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,WAAW,EAAE,CAAC;IACnB,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACrF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,MAAM,CAAC,iBAAU,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,OAAO,CAAC;IACnB,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,OAAO,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,KAAK,EAAE,CAAC;QACR,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,KAAa;IAC9C,IAAI,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,KAAa;IACxC,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,iBAAU,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAgB,aAAa,CAAC,GAAW,EAAE,QAAuB;IAC9D,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,gBAAgB,CAAC,QAAQ,CAAC,IAAI,OAAO,EAAE,gBAAgB,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAClH,CAAC;IAED,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC9C,6FAA6F;IAC7F,yFAAyF;IACzF,MAAM,KAAK,GAAG,UAAU,IAAI,OAAO,CAAC;IAEpC,IAAI,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACtC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,QAAQ,GAAG,IAAI,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACrE,WAAW,GAAG,cAAc,CAAC;QAC7B,QAAQ,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI,gBAAgB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAE/C,8FAA8F;IAC9F,wFAAwF;IACxF,MAAM,kBAAkB,GAAG,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,MAAM,IAAI,CAAC,UAAU,CAAC;IAChF,IAAI,gBAAgB,GAAG,CAAC,IAAI,kBAAkB,IAAI,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,cAAc,EAAE,CAAC;QAC7G,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,KAAK,CAAC;YACb,WAAW,GAAG,cAAc,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;YAChB,gBAAgB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;AAC7D,CAAC"}
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Header extraction helpers.
3
+ *
4
+ * Nothing in the pipeline may depend on a header being present. Every accessor returns null rather
5
+ * than throwing, and every decode is wrapped so that a malformed encoded word cannot abort a parse.
6
+ */
7
+ import { type HeaderEntry, type MessageHeaders } from './model';
8
+ /** Decode MIME encoded words, falling back to the raw value on any failure. */
9
+ export declare function decodeWordsSafe(value: string): string;
10
+ /** Parse a structured header value such as Content-Type into a value plus parameters. */
11
+ export declare function parseHeaderValueSafe(value: string): {
12
+ value: string;
13
+ params: Record<string, string>;
14
+ };
15
+ /**
16
+ * Convert mailsplit's raw header lines into normalised entries.
17
+ *
18
+ * mailsplit returns `{ key, line }` where `line` still contains the field name and any folded
19
+ * continuations joined with CRLF. Lines with no colon at all are kept with an empty key so that
20
+ * malformed input is still visible to diagnostics.
21
+ */
22
+ export declare function toHeaderEntries(list: readonly {
23
+ key: string;
24
+ line: string;
25
+ }[]): HeaderEntry[];
26
+ /** Strip angle brackets and surrounding whitespace from a Message-ID style value. */
27
+ export declare function normaliseMessageId(value: string | null): string | null;
28
+ /** Extract every angle bracketed id from a References style header. */
29
+ export declare function parseReferences(value: string | null): string[];
30
+ /** Collect the headers the analysis pipeline uses. All fields are optional by design. */
31
+ export declare function extractMessageHeaders(entries: readonly HeaderEntry[]): MessageHeaders;
32
+ //# sourceMappingURL=headers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/mime/headers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,SAAS,CAAC;AAEhE,+EAA+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMrD;AAED,yFAAyF;AACzF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAOrG;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,SAAS;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,WAAW,EAAE,CAY7F;AAqBD,qFAAqF;AACrF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOtE;AAED,uEAAuE;AACvE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,EAAE,CAa9D;AAUD,yFAAyF;AACzF,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,cAAc,CAoBrF"}