@postalsys/email-content 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +288 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/analyze.d.ts +91 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +274 -0
- package/dist/analyze.js.map +1 -0
- package/dist/annotate/client.d.ts +64 -0
- package/dist/annotate/client.d.ts.map +1 -0
- package/dist/annotate/client.js +139 -0
- package/dist/annotate/client.js.map +1 -0
- package/dist/annotate/llm.d.ts +46 -0
- package/dist/annotate/llm.d.ts.map +1 -0
- package/dist/annotate/llm.js +107 -0
- package/dist/annotate/llm.js.map +1 -0
- package/dist/annotate/prompt.d.ts +38 -0
- package/dist/annotate/prompt.d.ts.map +1 -0
- package/dist/annotate/prompt.js +165 -0
- package/dist/annotate/prompt.js.map +1 -0
- package/dist/annotate/store.d.ts +43 -0
- package/dist/annotate/store.d.ts.map +1 -0
- package/dist/annotate/store.js +46 -0
- package/dist/annotate/store.js.map +1 -0
- package/dist/canonical/html.d.ts +33 -0
- package/dist/canonical/html.d.ts.map +1 -0
- package/dist/canonical/html.js +156 -0
- package/dist/canonical/html.js.map +1 -0
- package/dist/canonical/text.d.ts +75 -0
- package/dist/canonical/text.d.ts.map +1 -0
- package/dist/canonical/text.js +217 -0
- package/dist/canonical/text.js.map +1 -0
- package/dist/classify/confidence.d.ts +38 -0
- package/dist/classify/confidence.d.ts.map +1 -0
- package/dist/classify/confidence.js +59 -0
- package/dist/classify/confidence.js.map +1 -0
- package/dist/classify/reconcile.d.ts +52 -0
- package/dist/classify/reconcile.d.ts.map +1 -0
- package/dist/classify/reconcile.js +100 -0
- package/dist/classify/reconcile.js.map +1 -0
- package/dist/classify/rules.d.ts +51 -0
- package/dist/classify/rules.d.ts.map +1 -0
- package/dist/classify/rules.js +648 -0
- package/dist/classify/rules.js.map +1 -0
- package/dist/cli/annotate.d.ts +23 -0
- package/dist/cli/annotate.d.ts.map +1 -0
- package/dist/cli/annotate.js +328 -0
- package/dist/cli/annotate.js.map +1 -0
- package/dist/cli/args.d.ts +17 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +72 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/benchmark.d.ts +13 -0
- package/dist/cli/benchmark.d.ts.map +1 -0
- package/dist/cli/benchmark.js +172 -0
- package/dist/cli/benchmark.js.map +1 -0
- package/dist/cli/corpus.d.ts +9 -0
- package/dist/cli/corpus.d.ts.map +1 -0
- package/dist/cli/corpus.js +152 -0
- package/dist/cli/corpus.js.map +1 -0
- package/dist/cli/disagreements.d.ts +14 -0
- package/dist/cli/disagreements.d.ts.map +1 -0
- package/dist/cli/disagreements.js +133 -0
- package/dist/cli/disagreements.js.map +1 -0
- package/dist/cli/evaluate.d.ts +15 -0
- package/dist/cli/evaluate.d.ts.map +1 -0
- package/dist/cli/evaluate.js +215 -0
- package/dist/cli/evaluate.js.map +1 -0
- package/dist/cli/extract.d.ts +10 -0
- package/dist/cli/extract.d.ts.map +1 -0
- package/dist/cli/extract.js +139 -0
- package/dist/cli/extract.js.map +1 -0
- package/dist/cli/main.d.ts +9 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +147 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/corpus/analyze.d.ts +66 -0
- package/dist/corpus/analyze.d.ts.map +1 -0
- package/dist/corpus/analyze.js +214 -0
- package/dist/corpus/analyze.js.map +1 -0
- package/dist/corpus/migrations/index.d.ts +14 -0
- package/dist/corpus/migrations/index.d.ts.map +1 -0
- package/dist/corpus/migrations/index.js +158 -0
- package/dist/corpus/migrations/index.js.map +1 -0
- package/dist/corpus/sample.d.ts +66 -0
- package/dist/corpus/sample.d.ts.map +1 -0
- package/dist/corpus/sample.js +98 -0
- package/dist/corpus/sample.js.map +1 -0
- package/dist/corpus/scanner.d.ts +79 -0
- package/dist/corpus/scanner.d.ts.map +1 -0
- package/dist/corpus/scanner.js +256 -0
- package/dist/corpus/scanner.js.map +1 -0
- package/dist/corpus/stats.d.ts +76 -0
- package/dist/corpus/stats.d.ts.map +1 -0
- package/dist/corpus/stats.js +240 -0
- package/dist/corpus/stats.js.map +1 -0
- package/dist/corpus/store.d.ts +67 -0
- package/dist/corpus/store.d.ts.map +1 -0
- package/dist/corpus/store.js +196 -0
- package/dist/corpus/store.js.map +1 -0
- package/dist/corpus/thread-index.d.ts +31 -0
- package/dist/corpus/thread-index.d.ts.map +1 -0
- package/dist/corpus/thread-index.js +182 -0
- package/dist/corpus/thread-index.js.map +1 -0
- package/dist/corpus/worker.d.ts +21 -0
- package/dist/corpus/worker.d.ts.map +1 -0
- package/dist/corpus/worker.js +74 -0
- package/dist/corpus/worker.js.map +1 -0
- package/dist/detect/client.d.ts +28 -0
- package/dist/detect/client.d.ts.map +1 -0
- package/dist/detect/client.js +121 -0
- package/dist/detect/client.js.map +1 -0
- package/dist/detect/disclaimer.d.ts +12 -0
- package/dist/detect/disclaimer.d.ts.map +1 -0
- package/dist/detect/disclaimer.js +28 -0
- package/dist/detect/disclaimer.js.map +1 -0
- package/dist/detect/forward-header.d.ts +10 -0
- package/dist/detect/forward-header.d.ts.map +1 -0
- package/dist/detect/forward-header.js +35 -0
- package/dist/detect/forward-header.js.map +1 -0
- package/dist/detect/patterns/disclaimers.json +113 -0
- package/dist/detect/patterns/forward-headers.json +40 -0
- package/dist/detect/patterns/html-clients.json +157 -0
- package/dist/detect/patterns/quoted-headers.json +41 -0
- package/dist/detect/patterns/reply-headers.json +139 -0
- package/dist/detect/patterns/signatures.json +58 -0
- package/dist/detect/patterns.d.ts +43 -0
- package/dist/detect/patterns.d.ts.map +1 -0
- package/dist/detect/patterns.js +125 -0
- package/dist/detect/patterns.js.map +1 -0
- package/dist/detect/reply-header.d.ts +44 -0
- package/dist/detect/reply-header.d.ts.map +1 -0
- package/dist/detect/reply-header.js +134 -0
- package/dist/detect/reply-header.js.map +1 -0
- package/dist/detect/signature.d.ts +28 -0
- package/dist/detect/signature.d.ts.map +1 -0
- package/dist/detect/signature.js +77 -0
- package/dist/detect/signature.js.map +1 -0
- package/dist/detect/traffic.d.ts +26 -0
- package/dist/detect/traffic.d.ts.map +1 -0
- package/dist/detect/traffic.js +98 -0
- package/dist/detect/traffic.js.map +1 -0
- package/dist/eval/metrics.d.ts +87 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +185 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/index.d.ts +54 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +163 -0
- package/dist/index.js.map +1 -0
- package/dist/match/history.d.ts +62 -0
- package/dist/match/history.d.ts.map +1 -0
- package/dist/match/history.js +128 -0
- package/dist/match/history.js.map +1 -0
- package/dist/match/normalize.d.ts +33 -0
- package/dist/match/normalize.d.ts.map +1 -0
- package/dist/match/normalize.js +63 -0
- package/dist/match/normalize.js.map +1 -0
- package/dist/match/shingle.d.ts +44 -0
- package/dist/match/shingle.d.ts.map +1 -0
- package/dist/match/shingle.js +78 -0
- package/dist/match/shingle.js.map +1 -0
- package/dist/mime/charset.d.ts +33 -0
- package/dist/mime/charset.d.ts.map +1 -0
- package/dist/mime/charset.js +118 -0
- package/dist/mime/charset.js.map +1 -0
- package/dist/mime/headers.d.ts +32 -0
- package/dist/mime/headers.d.ts.map +1 -0
- package/dist/mime/headers.js +127 -0
- package/dist/mime/headers.js.map +1 -0
- package/dist/mime/model.d.ts +111 -0
- package/dist/mime/model.d.ts.map +1 -0
- package/dist/mime/model.js +17 -0
- package/dist/mime/model.js.map +1 -0
- package/dist/mime/parse.d.ts +27 -0
- package/dist/mime/parse.d.ts.map +1 -0
- package/dist/mime/parse.js +343 -0
- package/dist/mime/parse.js.map +1 -0
- package/dist/segment/features.d.ts +29 -0
- package/dist/segment/features.d.ts.map +1 -0
- package/dist/segment/features.js +102 -0
- package/dist/segment/features.js.map +1 -0
- package/dist/segment/html-blocks.d.ts +28 -0
- package/dist/segment/html-blocks.d.ts.map +1 -0
- package/dist/segment/html-blocks.js +596 -0
- package/dist/segment/html-blocks.js.map +1 -0
- package/dist/segment/text-blocks.d.ts +30 -0
- package/dist/segment/text-blocks.d.ts.map +1 -0
- package/dist/segment/text-blocks.js +204 -0
- package/dist/segment/text-blocks.js.map +1 -0
- package/dist/segment/types.d.ts +88 -0
- package/dist/segment/types.d.ts.map +1 -0
- package/dist/segment/types.js +11 -0
- package/dist/segment/types.js.map +1 -0
- package/dist/types.d.ts +85 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +17 -0
- package/dist/types.js.map +1 -0
- package/dist/util/hash.d.ts +49 -0
- package/dist/util/hash.d.ts.map +1 -0
- package/dist/util/hash.js +0 -0
- package/dist/util/hash.js.map +1 -0
- package/dist/util/lang.d.ts +25 -0
- package/dist/util/lang.d.ts.map +1 -0
- package/dist/util/lang.js +135 -0
- package/dist/util/lang.js.map +1 -0
- package/dist/util/worker-pool.d.ts +51 -0
- package/dist/util/worker-pool.d.ts.map +1 -0
- package/dist/util/worker-pool.js +155 -0
- package/dist/util/worker-pool.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Stage 5: matching blocks against messages that already existed.
|
|
4
|
+
*
|
|
5
|
+
* This is the strongest evidence the pipeline can have. Every other signal is an inference from
|
|
6
|
+
* markup or wording: an attribution line suggests a quote follows, a blockquote suggests its
|
|
7
|
+
* contents are old. A match against the actual parent message is not an inference. The text
|
|
8
|
+
* demonstrably existed before this message was written, so it cannot be something the sender just
|
|
9
|
+
* wrote.
|
|
10
|
+
*
|
|
11
|
+
* In the working corpus 90.9 percent of replies resolve their In-Reply-To to a message that is also
|
|
12
|
+
* present, which makes this unusually available here. It remains strictly optional: extraction
|
|
13
|
+
* works with no history at all, and history only ever adds evidence.
|
|
14
|
+
*
|
|
15
|
+
* The matcher is deliberately conservative in one direction. A match can promote a block to
|
|
16
|
+
* REPLY_HISTORY, which hides it, so short blocks are never matched, and a high containment
|
|
17
|
+
* threshold is required. A non match never demotes anything.
|
|
18
|
+
*/
|
|
19
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.buildHistoryDocument = buildHistoryDocument;
|
|
21
|
+
exports.buildHistoryDocumentFromContent = buildHistoryDocumentFromContent;
|
|
22
|
+
exports.matchBlocks = matchBlocks;
|
|
23
|
+
exports.matchedIndices = matchedIndices;
|
|
24
|
+
const parse_1 = require("../mime/parse");
|
|
25
|
+
const text_1 = require("../canonical/text");
|
|
26
|
+
const html_1 = require("../canonical/html");
|
|
27
|
+
const shingle_1 = require("./shingle");
|
|
28
|
+
const normalize_1 = require("./normalize");
|
|
29
|
+
/**
|
|
30
|
+
* Build a searchable document from a raw earlier message.
|
|
31
|
+
*
|
|
32
|
+
* Both representations are folded into one text projection. Which representation the earlier
|
|
33
|
+
* message used has nothing to do with which one the reply quoted from, and clients routinely quote
|
|
34
|
+
* the HTML of a message into the plain text of a reply.
|
|
35
|
+
*/
|
|
36
|
+
async function buildHistoryDocument(raw, id, isParent) {
|
|
37
|
+
const buffer = typeof raw === 'string' ? Buffer.from(raw) : raw;
|
|
38
|
+
const message = await (0, parse_1.parseMessage)(buffer);
|
|
39
|
+
const parts = [];
|
|
40
|
+
for (const part of message.parts) {
|
|
41
|
+
if (!part.decoded || part.disposition === 'attachment') {
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
if (part.contentType === 'text/plain') {
|
|
45
|
+
parts.push((0, text_1.canonicaliseText)(part.decoded.text).text);
|
|
46
|
+
}
|
|
47
|
+
else if (part.contentType === 'text/html') {
|
|
48
|
+
parts.push((0, html_1.stripTagsToText)(part.decoded.text));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const normalised = (0, normalize_1.normaliseForMatch)(parts.join('\n'));
|
|
52
|
+
return { id, normalised, index: (0, shingle_1.buildShingleIndex)(normalised), isParent };
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Build a searchable document from already parsed content.
|
|
56
|
+
*
|
|
57
|
+
* The counterpart to buildHistoryDocument for callers that never had a raw message. Both
|
|
58
|
+
* representations are folded together for the same reason: which one the earlier message used has
|
|
59
|
+
* nothing to do with which one the reply quoted from.
|
|
60
|
+
*/
|
|
61
|
+
function buildHistoryDocumentFromContent(text, html, id, isParent) {
|
|
62
|
+
const parts = [];
|
|
63
|
+
if (text) {
|
|
64
|
+
parts.push((0, text_1.canonicaliseText)(text).text);
|
|
65
|
+
}
|
|
66
|
+
if (html) {
|
|
67
|
+
parts.push((0, html_1.stripTagsToText)(html));
|
|
68
|
+
}
|
|
69
|
+
const normalised = (0, normalize_1.normaliseForMatch)(parts.join('\n'));
|
|
70
|
+
return { id, normalised, index: (0, shingle_1.buildShingleIndex)(normalised), isParent };
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Match blocks against known history.
|
|
74
|
+
*
|
|
75
|
+
* Returns one match per block at most, preferring the direct parent over older ancestors, and
|
|
76
|
+
* higher containment over lower.
|
|
77
|
+
*/
|
|
78
|
+
function matchBlocks(blocks, history) {
|
|
79
|
+
if (!history.length) {
|
|
80
|
+
return [];
|
|
81
|
+
}
|
|
82
|
+
const matches = [];
|
|
83
|
+
for (const block of blocks) {
|
|
84
|
+
const normalised = (0, normalize_1.normaliseForMatch)(block.content);
|
|
85
|
+
if (!normalised) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const index = (0, shingle_1.buildShingleIndex)(normalised);
|
|
89
|
+
if (!(0, shingle_1.isMatchable)(index)) {
|
|
90
|
+
// Too short to judge. "Thanks" appearing in the parent proves nothing about this block.
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
let best = null;
|
|
94
|
+
for (const document of history) {
|
|
95
|
+
// Tier one: the normalised text appears verbatim in the earlier message.
|
|
96
|
+
const exact = document.normalised.includes(normalised);
|
|
97
|
+
const score = exact ? 1 : (0, shingle_1.containment)(index, document.index).containment;
|
|
98
|
+
if (score < shingle_1.CONTAINMENT_THRESHOLD) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
const candidate = {
|
|
102
|
+
blockIndex: block.index,
|
|
103
|
+
documentId: document.id,
|
|
104
|
+
isParent: document.isParent,
|
|
105
|
+
containment: score,
|
|
106
|
+
exact
|
|
107
|
+
};
|
|
108
|
+
if (!best || betterMatch(candidate, best)) {
|
|
109
|
+
best = candidate;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (best) {
|
|
113
|
+
matches.push(best);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return matches;
|
|
117
|
+
}
|
|
118
|
+
function betterMatch(candidate, current) {
|
|
119
|
+
if (candidate.isParent !== current.isParent) {
|
|
120
|
+
return candidate.isParent;
|
|
121
|
+
}
|
|
122
|
+
return candidate.containment > current.containment;
|
|
123
|
+
}
|
|
124
|
+
/** Convenience view for the classifier, which only needs to know which indices matched. */
|
|
125
|
+
function matchedIndices(matches) {
|
|
126
|
+
return new Set(matches.map(match => match.blockIndex));
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/match/history.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;AAmCH,oDAkBC;AASD,0EAUC;AAQD,kCA4CC;AAUD,wCAEC;AAtID,yCAA6C;AAC7C,4CAAqD;AACrD,4CAAoD;AAEpD,uCAAkH;AAClH,2CAAgD;AAqBhD;;;;;;GAMG;AACI,KAAK,UAAU,oBAAoB,CAAC,GAAoB,EAAE,EAAU,EAAE,QAAiB;IAC1F,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAChE,MAAM,OAAO,GAAG,MAAM,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACrD,SAAS;QACb,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,+BAA+B,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU,EAAE,QAAiB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,EAAE,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,IAAA,sBAAe,EAAC,IAAI,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,IAAA,2BAAiB,EAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,MAAwB,EAAE,OAAmC;IACrF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAA,6BAAiB,EAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAA,2BAAiB,EAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAA,qBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;YACtB,wFAAwF;YACxF,SAAS;QACb,CAAC;QAED,IAAI,IAAI,GAAsB,IAAI,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC7B,yEAAyE;YACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,qBAAW,EAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC;YACzE,IAAI,KAAK,GAAG,+BAAqB,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAe;gBAC1B,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,KAAK;gBAClB,KAAK;aACR,CAAC;YACF,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBACxC,IAAI,GAAG,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,SAAqB,EAAE,OAAmB;IAC3D,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,SAAS,CAAC,QAAQ,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;AACvD,CAAC;AAED,2FAA2F;AAC3F,SAAgB,cAAc,CAAC,OAA8B;IACzD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Normalisation for history matching.
|
|
3
|
+
*
|
|
4
|
+
* The same sentence, quoted by a mail client, is rarely byte identical to the original. It has been
|
|
5
|
+
* re-wrapped, prefixed with ">", had its entities decoded or re-encoded, had non-breaking spaces
|
|
6
|
+
* substituted, had smart quotes swapped for straight ones, and may have been transcoded through a
|
|
7
|
+
* legacy charset and back.
|
|
8
|
+
*
|
|
9
|
+
* Matching therefore happens on a normalised projection: aggressive enough to survive all of that,
|
|
10
|
+
* conservative enough that two genuinely different sentences do not collide. Note the asymmetry in
|
|
11
|
+
* consequences here is the opposite of the rest of the pipeline: a false match causes content to be
|
|
12
|
+
* hidden, so this normalisation deliberately stops short of stemming, case folding of proper nouns,
|
|
13
|
+
* or anything else that would let distinct text collapse together.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Normalise text for comparison.
|
|
17
|
+
*
|
|
18
|
+
* Quote prefixes are stripped, whitespace is collapsed to single spaces, punctuation variants are
|
|
19
|
+
* folded, and the result is lowercased. Line structure is deliberately discarded, because re-wrapping
|
|
20
|
+
* is the single most common transformation a quoted message undergoes.
|
|
21
|
+
*/
|
|
22
|
+
export declare function normaliseForMatch(text: string): string;
|
|
23
|
+
/** Word tokens used for shingling and alignment. */
|
|
24
|
+
export declare function tokenise(normalised: string): string[];
|
|
25
|
+
/**
|
|
26
|
+
* Minimum tokens before a block is worth matching at all.
|
|
27
|
+
*
|
|
28
|
+
* Short blocks match by coincidence. "Thanks", "Yes", "Best regards" and "OK" appear in almost
|
|
29
|
+
* every thread, so a match on them says nothing about whether this particular text was quoted. Nine
|
|
30
|
+
* tokens is roughly a full sentence.
|
|
31
|
+
*/
|
|
32
|
+
export declare const MIN_MATCH_TOKENS = 9;
|
|
33
|
+
//# sourceMappingURL=normalize.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/match/normalize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAaH;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYtD;AAED,oDAAoD;AACpD,wBAAgB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAErD;AAED;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,IAAI,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Normalisation for history matching.
|
|
4
|
+
*
|
|
5
|
+
* The same sentence, quoted by a mail client, is rarely byte identical to the original. It has been
|
|
6
|
+
* re-wrapped, prefixed with ">", had its entities decoded or re-encoded, had non-breaking spaces
|
|
7
|
+
* substituted, had smart quotes swapped for straight ones, and may have been transcoded through a
|
|
8
|
+
* legacy charset and back.
|
|
9
|
+
*
|
|
10
|
+
* Matching therefore happens on a normalised projection: aggressive enough to survive all of that,
|
|
11
|
+
* conservative enough that two genuinely different sentences do not collide. Note the asymmetry in
|
|
12
|
+
* consequences here is the opposite of the rest of the pipeline: a false match causes content to be
|
|
13
|
+
* hidden, so this normalisation deliberately stops short of stemming, case folding of proper nouns,
|
|
14
|
+
* or anything else that would let distinct text collapse together.
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.MIN_MATCH_TOKENS = void 0;
|
|
18
|
+
exports.normaliseForMatch = normaliseForMatch;
|
|
19
|
+
exports.tokenise = tokenise;
|
|
20
|
+
/** Punctuation and whitespace variants that clients substitute freely. */
|
|
21
|
+
const REPLACEMENTS = [
|
|
22
|
+
[/[\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]/g, ' '], // exotic spaces
|
|
23
|
+
[/[\u200b-\u200d\ufeff\u00ad]/g, ''], // zero width and soft hyphens
|
|
24
|
+
[/[‘’‚‛′‵]/g, "'"], // single quotes
|
|
25
|
+
[/[“”„‟″‶]/g, '"'], // double quotes
|
|
26
|
+
[/[‐-―−]/g, '-'], // dashes
|
|
27
|
+
[/[…]/g, '...'],
|
|
28
|
+
[/[•·⁃]/g, '*'] // bullets
|
|
29
|
+
];
|
|
30
|
+
/**
|
|
31
|
+
* Normalise text for comparison.
|
|
32
|
+
*
|
|
33
|
+
* Quote prefixes are stripped, whitespace is collapsed to single spaces, punctuation variants are
|
|
34
|
+
* folded, and the result is lowercased. Line structure is deliberately discarded, because re-wrapping
|
|
35
|
+
* is the single most common transformation a quoted message undergoes.
|
|
36
|
+
*/
|
|
37
|
+
function normaliseForMatch(text) {
|
|
38
|
+
let out = text;
|
|
39
|
+
try {
|
|
40
|
+
out = out.normalize('NFC');
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// Unnormalisable input still compares usefully.
|
|
44
|
+
}
|
|
45
|
+
out = out.replace(/^[ \t]{0,3}(?:>[ \t]?)+/gm, '');
|
|
46
|
+
for (const [pattern, replacement] of REPLACEMENTS) {
|
|
47
|
+
out = out.replace(pattern, replacement);
|
|
48
|
+
}
|
|
49
|
+
return out.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
50
|
+
}
|
|
51
|
+
/** Word tokens used for shingling and alignment. */
|
|
52
|
+
function tokenise(normalised) {
|
|
53
|
+
return normalised.split(/[^\p{L}\p{N}]+/u).filter(token => token.length > 0);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Minimum tokens before a block is worth matching at all.
|
|
57
|
+
*
|
|
58
|
+
* Short blocks match by coincidence. "Thanks", "Yes", "Best regards" and "OK" appear in almost
|
|
59
|
+
* every thread, so a match on them says nothing about whether this particular text was quoted. Nine
|
|
60
|
+
* tokens is roughly a full sentence.
|
|
61
|
+
*/
|
|
62
|
+
exports.MIN_MATCH_TOKENS = 9;
|
|
63
|
+
//# sourceMappingURL=normalize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/match/normalize.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;AAoBH,8CAYC;AAGD,4BAEC;AAnCD,0EAA0E;AAC1E,MAAM,YAAY,GAAgC;IAC9C,CAAC,gDAAgD,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACzE,CAAC,8BAA8B,EAAE,EAAE,CAAC,EAAE,8BAA8B;IACpE,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACpC,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,gBAAgB;IACpC,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,SAAS;IAC3B,CAAC,MAAM,EAAE,KAAK,CAAC;IACf,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU;CAC7B,CAAC;AAEF;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,IAAY;IAC1C,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,IAAI,CAAC;QACD,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACL,gDAAgD;IACpD,CAAC;IACD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC;QAChD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACzD,CAAC;AAED,oDAAoD;AACpD,SAAgB,QAAQ,CAAC,UAAkB;IACvC,OAAO,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjF,CAAC;AAED;;;;;;GAMG;AACU,QAAA,gBAAgB,GAAG,CAAC,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token shingle matching with rolling hashes.
|
|
3
|
+
*
|
|
4
|
+
* Stage 5's middle tier. Exact normalised comparison catches text that was quoted verbatim;
|
|
5
|
+
* shingles catch text that was quoted with an edit, a truncation, or an interleaved insertion,
|
|
6
|
+
* which clients and people both do.
|
|
7
|
+
*
|
|
8
|
+
* A shingle is a window of consecutive tokens. Two texts sharing most of their shingles share most
|
|
9
|
+
* of their content regardless of where it moved to. The hash is rolling, so building the index for
|
|
10
|
+
* a parent message costs one pass over its tokens.
|
|
11
|
+
*/
|
|
12
|
+
/** Tokens per shingle. Long enough to be specific, short enough to survive small edits. */
|
|
13
|
+
export declare const SHINGLE_SIZE = 5;
|
|
14
|
+
export interface ShingleIndex {
|
|
15
|
+
hashes: Set<number>;
|
|
16
|
+
tokenCount: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Build a shingle index over normalised text.
|
|
20
|
+
*
|
|
21
|
+
* The rolling hash is a simple polynomial over per token 32 bit hashes. Collisions are possible and
|
|
22
|
+
* harmless at this scale: a collision inflates a similarity score slightly, and the containment
|
|
23
|
+
* threshold plus the corroboration rules downstream absorb that.
|
|
24
|
+
*/
|
|
25
|
+
export declare function buildShingleIndex(normalised: string): ShingleIndex;
|
|
26
|
+
export interface ContainmentResult {
|
|
27
|
+
/** Share of the block's shingles that also appear in the reference. */
|
|
28
|
+
containment: number;
|
|
29
|
+
/** Shingles compared. Zero means the block was too short to judge. */
|
|
30
|
+
shingles: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* How much of `block` is contained in `reference`.
|
|
34
|
+
*
|
|
35
|
+
* Containment rather than Jaccard similarity, deliberately: a quoted paragraph is a small part of a
|
|
36
|
+
* large parent message, so symmetric similarity would be near zero for a perfect quote. What
|
|
37
|
+
* matters is whether everything in this block already existed.
|
|
38
|
+
*/
|
|
39
|
+
export declare function containment(block: ShingleIndex, reference: ShingleIndex): ContainmentResult;
|
|
40
|
+
/** Containment at or above this counts as a match, once the block is long enough to judge. */
|
|
41
|
+
export declare const CONTAINMENT_THRESHOLD = 0.8;
|
|
42
|
+
/** True when a block is long enough for a containment score to mean anything. */
|
|
43
|
+
export declare function isMatchable(index: ShingleIndex): boolean;
|
|
44
|
+
//# sourceMappingURL=shingle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shingle.d.ts","sourceRoot":"","sources":["../../src/match/shingle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,2FAA2F;AAC3F,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,CAclE;AAYD,MAAM,WAAW,iBAAiB;IAC9B,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,GAAG,iBAAiB,CAW3F;AAED,8FAA8F;AAC9F,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAExD"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Token shingle matching with rolling hashes.
|
|
4
|
+
*
|
|
5
|
+
* Stage 5's middle tier. Exact normalised comparison catches text that was quoted verbatim;
|
|
6
|
+
* shingles catch text that was quoted with an edit, a truncation, or an interleaved insertion,
|
|
7
|
+
* which clients and people both do.
|
|
8
|
+
*
|
|
9
|
+
* A shingle is a window of consecutive tokens. Two texts sharing most of their shingles share most
|
|
10
|
+
* of their content regardless of where it moved to. The hash is rolling, so building the index for
|
|
11
|
+
* a parent message costs one pass over its tokens.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.CONTAINMENT_THRESHOLD = exports.SHINGLE_SIZE = void 0;
|
|
15
|
+
exports.buildShingleIndex = buildShingleIndex;
|
|
16
|
+
exports.containment = containment;
|
|
17
|
+
exports.isMatchable = isMatchable;
|
|
18
|
+
const hash_1 = require("../util/hash");
|
|
19
|
+
const normalize_1 = require("./normalize");
|
|
20
|
+
/** Tokens per shingle. Long enough to be specific, short enough to survive small edits. */
|
|
21
|
+
exports.SHINGLE_SIZE = 5;
|
|
22
|
+
/**
|
|
23
|
+
* Build a shingle index over normalised text.
|
|
24
|
+
*
|
|
25
|
+
* The rolling hash is a simple polynomial over per token 32 bit hashes. Collisions are possible and
|
|
26
|
+
* harmless at this scale: a collision inflates a similarity score slightly, and the containment
|
|
27
|
+
* threshold plus the corroboration rules downstream absorb that.
|
|
28
|
+
*/
|
|
29
|
+
function buildShingleIndex(normalised) {
|
|
30
|
+
const tokens = (0, normalize_1.tokenise)(normalised);
|
|
31
|
+
const hashes = new Set();
|
|
32
|
+
if (tokens.length < exports.SHINGLE_SIZE) {
|
|
33
|
+
// Short text is indexed as a single shingle so it can still match exactly.
|
|
34
|
+
if (tokens.length > 0) {
|
|
35
|
+
hashes.add(hashTokens(tokens, 0, tokens.length));
|
|
36
|
+
}
|
|
37
|
+
return { hashes, tokenCount: tokens.length };
|
|
38
|
+
}
|
|
39
|
+
for (let i = 0; i + exports.SHINGLE_SIZE <= tokens.length; i++) {
|
|
40
|
+
hashes.add(hashTokens(tokens, i, exports.SHINGLE_SIZE));
|
|
41
|
+
}
|
|
42
|
+
return { hashes, tokenCount: tokens.length };
|
|
43
|
+
}
|
|
44
|
+
function hashTokens(tokens, start, length) {
|
|
45
|
+
let hash = 0x811c9dc5;
|
|
46
|
+
for (let i = start; i < start + length; i++) {
|
|
47
|
+
const token = tokens[i];
|
|
48
|
+
hash = (Math.imul(hash, 31) + (0, hash_1.fnv1a32)(token)) >>> 0;
|
|
49
|
+
hash = (hash ^ (0, hash_1.fnv1a32b)(token)) >>> 0;
|
|
50
|
+
}
|
|
51
|
+
return hash >>> 0;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* How much of `block` is contained in `reference`.
|
|
55
|
+
*
|
|
56
|
+
* Containment rather than Jaccard similarity, deliberately: a quoted paragraph is a small part of a
|
|
57
|
+
* large parent message, so symmetric similarity would be near zero for a perfect quote. What
|
|
58
|
+
* matters is whether everything in this block already existed.
|
|
59
|
+
*/
|
|
60
|
+
function containment(block, reference) {
|
|
61
|
+
if (block.hashes.size === 0) {
|
|
62
|
+
return { containment: 0, shingles: 0 };
|
|
63
|
+
}
|
|
64
|
+
let hits = 0;
|
|
65
|
+
for (const hash of block.hashes) {
|
|
66
|
+
if (reference.hashes.has(hash)) {
|
|
67
|
+
hits++;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return { containment: hits / block.hashes.size, shingles: block.hashes.size };
|
|
71
|
+
}
|
|
72
|
+
/** Containment at or above this counts as a match, once the block is long enough to judge. */
|
|
73
|
+
exports.CONTAINMENT_THRESHOLD = 0.8;
|
|
74
|
+
/** True when a block is long enough for a containment score to mean anything. */
|
|
75
|
+
function isMatchable(index) {
|
|
76
|
+
return index.tokenCount >= normalize_1.MIN_MATCH_TOKENS;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=shingle.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shingle.js","sourceRoot":"","sources":["../../src/match/shingle.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAoBH,8CAcC;AA0BD,kCAWC;AAMD,kCAEC;AA7ED,uCAAiD;AACjD,2CAAyD;AAEzD,2FAA2F;AAC9E,QAAA,YAAY,GAAG,CAAC,CAAC;AAO9B;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,UAAkB;IAChD,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,UAAU,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,GAAG,oBAAY,EAAE,CAAC;QAC/B,2EAA2E;QAC3E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAY,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,oBAAY,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,KAAa,EAAE,MAAc;IACxE,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,KAAK,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;QAClC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAA,cAAO,EAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,GAAG,CAAC,IAAI,GAAG,IAAA,eAAQ,EAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACtB,CAAC;AASD;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,KAAmB,EAAE,SAAuB;IACpE,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3C,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,IAAI,EAAE,CAAC;QACX,CAAC;IACL,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClF,CAAC;AAED,8FAA8F;AACjF,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC,iFAAiF;AACjF,SAAgB,WAAW,CAAC,KAAmB;IAC3C,OAAO,KAAK,CAAC,UAAU,IAAI,4BAAgB,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Charset decoding with an explicit, recorded fallback chain.
|
|
3
|
+
*
|
|
4
|
+
* Measured on the working corpus: about 14 percent of text parts declare something other than
|
|
5
|
+
* utf-8, including iso-8859-1, iso-8859-15, windows-1257, windows-1252 and iso-2022-jp. A silent
|
|
6
|
+
* fallback would corrupt those messages without anyone noticing, so every downgrade is reported
|
|
7
|
+
* back to the caller.
|
|
8
|
+
*
|
|
9
|
+
* Chain:
|
|
10
|
+
* 1. Normalise the declared label through libmime (handles aliases such as "unicode-1-1-utf-8").
|
|
11
|
+
* 2. Node's built in TextDecoder, which covers the WHATWG encoding set including the legacy
|
|
12
|
+
* single byte and CJK encodings.
|
|
13
|
+
* 3. libmime's iconv-lite backed decoder, which covers labels TextDecoder rejects (utf-7 and a
|
|
14
|
+
* long tail of aliases).
|
|
15
|
+
* 4. windows-1252, which cannot fail, flagged as a downgrade.
|
|
16
|
+
*/
|
|
17
|
+
export interface CharsetDecodeResult {
|
|
18
|
+
text: string;
|
|
19
|
+
charsetUsed: string;
|
|
20
|
+
replacementChars: number;
|
|
21
|
+
/** True when the declared charset could not be honoured. */
|
|
22
|
+
fellBack: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Normalise a raw charset label. Returns null when the label is empty or meaningless. */
|
|
25
|
+
export declare function normaliseCharset(label: string | null | undefined): string | null;
|
|
26
|
+
/**
|
|
27
|
+
* Decode a byte buffer using the declared charset, falling back predictably.
|
|
28
|
+
*
|
|
29
|
+
* @param buf raw bytes, already transfer decoded
|
|
30
|
+
* @param declared charset label from the Content-Type header, or null when absent
|
|
31
|
+
*/
|
|
32
|
+
export declare function decodeCharset(buf: Buffer, declared: string | null): CharsetDecodeResult;
|
|
33
|
+
//# sourceMappingURL=charset.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"charset.d.ts","sourceRoot":"","sources":["../../src/mime/charset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAaH,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,EAAE,OAAO,CAAC;CACrB;AAED,0FAA0F;AAC1F,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAgBhF;AA6BD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAyCvF"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Charset decoding with an explicit, recorded fallback chain.
|
|
4
|
+
*
|
|
5
|
+
* Measured on the working corpus: about 14 percent of text parts declare something other than
|
|
6
|
+
* utf-8, including iso-8859-1, iso-8859-15, windows-1257, windows-1252 and iso-2022-jp. A silent
|
|
7
|
+
* fallback would corrupt those messages without anyone noticing, so every downgrade is reported
|
|
8
|
+
* back to the caller.
|
|
9
|
+
*
|
|
10
|
+
* Chain:
|
|
11
|
+
* 1. Normalise the declared label through libmime (handles aliases such as "unicode-1-1-utf-8").
|
|
12
|
+
* 2. Node's built in TextDecoder, which covers the WHATWG encoding set including the legacy
|
|
13
|
+
* single byte and CJK encodings.
|
|
14
|
+
* 3. libmime's iconv-lite backed decoder, which covers labels TextDecoder rejects (utf-7 and a
|
|
15
|
+
* long tail of aliases).
|
|
16
|
+
* 4. windows-1252, which cannot fail, flagged as a downgrade.
|
|
17
|
+
*/
|
|
18
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
19
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
20
|
+
};
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.normaliseCharset = normaliseCharset;
|
|
23
|
+
exports.decodeCharset = decodeCharset;
|
|
24
|
+
const charset_1 = __importDefault(require("libmime/lib/charset"));
|
|
25
|
+
const REPLACEMENT = '�';
|
|
26
|
+
/**
|
|
27
|
+
* A utf-8 decode that produces more than this share of replacement characters is treated as a
|
|
28
|
+
* mislabelled single byte encoding rather than as damaged utf-8. Chosen so that a single bad byte
|
|
29
|
+
* in a long message does not trigger a switch, while genuinely mislabelled Latin-1 text does.
|
|
30
|
+
*/
|
|
31
|
+
const MOJIBAKE_RATIO = 0.02;
|
|
32
|
+
/** Normalise a raw charset label. Returns null when the label is empty or meaningless. */
|
|
33
|
+
function normaliseCharset(label) {
|
|
34
|
+
if (!label) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
const trimmed = label
|
|
38
|
+
.trim()
|
|
39
|
+
.replace(/^["']|["']$/g, '')
|
|
40
|
+
.toLowerCase();
|
|
41
|
+
if (!trimmed || trimmed === 'unknown' || trimmed === 'x-unknown' || trimmed === 'none') {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
try {
|
|
45
|
+
return String(charset_1.default.normalizeCharset(trimmed) || trimmed).toLowerCase();
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return trimmed;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function countReplacements(text) {
|
|
52
|
+
let count = 0;
|
|
53
|
+
let index = text.indexOf(REPLACEMENT);
|
|
54
|
+
while (index !== -1) {
|
|
55
|
+
count++;
|
|
56
|
+
index = text.indexOf(REPLACEMENT, index + 1);
|
|
57
|
+
}
|
|
58
|
+
return count;
|
|
59
|
+
}
|
|
60
|
+
function tryTextDecoder(buf, label) {
|
|
61
|
+
try {
|
|
62
|
+
return new TextDecoder(label, { fatal: false }).decode(buf);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function tryIconv(buf, label) {
|
|
69
|
+
try {
|
|
70
|
+
const decoded = charset_1.default.decode(buf, label);
|
|
71
|
+
return typeof decoded === 'string' ? decoded : null;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Decode a byte buffer using the declared charset, falling back predictably.
|
|
79
|
+
*
|
|
80
|
+
* @param buf raw bytes, already transfer decoded
|
|
81
|
+
* @param declared charset label from the Content-Type header, or null when absent
|
|
82
|
+
*/
|
|
83
|
+
function decodeCharset(buf, declared) {
|
|
84
|
+
if (buf.length === 0) {
|
|
85
|
+
return { text: '', charsetUsed: normaliseCharset(declared) || 'utf-8', replacementChars: 0, fellBack: false };
|
|
86
|
+
}
|
|
87
|
+
const normalised = normaliseCharset(declared);
|
|
88
|
+
// RFC 2045 says us-ascii when absent. Decoding as utf-8 is a strict superset for valid ascii
|
|
89
|
+
// and recovers undeclared utf-8, which is far more common in practice than strict ascii.
|
|
90
|
+
const label = normalised || 'utf-8';
|
|
91
|
+
let text = tryTextDecoder(buf, label);
|
|
92
|
+
let charsetUsed = label;
|
|
93
|
+
let fellBack = false;
|
|
94
|
+
if (text === null) {
|
|
95
|
+
text = tryIconv(buf, label);
|
|
96
|
+
fellBack = text !== null;
|
|
97
|
+
}
|
|
98
|
+
if (text === null) {
|
|
99
|
+
text = tryTextDecoder(buf, 'windows-1252') ?? buf.toString('latin1');
|
|
100
|
+
charsetUsed = 'windows-1252';
|
|
101
|
+
fellBack = true;
|
|
102
|
+
}
|
|
103
|
+
let replacementChars = countReplacements(text);
|
|
104
|
+
// Mislabelled single byte content declared as utf-8 is common. Only switch when the damage is
|
|
105
|
+
// widespread, so that one stray byte in an otherwise valid utf-8 message is left alone.
|
|
106
|
+
const looksLikeUtf8Label = label === 'utf-8' || label === 'utf8' || !normalised;
|
|
107
|
+
if (replacementChars > 0 && looksLikeUtf8Label && replacementChars / Math.max(text.length, 1) > MOJIBAKE_RATIO) {
|
|
108
|
+
const retry = tryTextDecoder(buf, 'windows-1252');
|
|
109
|
+
if (retry !== null) {
|
|
110
|
+
text = retry;
|
|
111
|
+
charsetUsed = 'windows-1252';
|
|
112
|
+
fellBack = true;
|
|
113
|
+
replacementChars = countReplacements(text);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return { text, charsetUsed, replacementChars, fellBack };
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=charset.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"charset.js","sourceRoot":"","sources":["../../src/mime/charset.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;;;AAsBH,4CAgBC;AAmCD,sCAyCC;AAhHD,kEAA6C;AAE7C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB;;;;GAIG;AACH,MAAM,cAAc,GAAG,IAAI,CAAC;AAU5B,0FAA0F;AAC1F,SAAgB,gBAAgB,CAAC,KAAgC;IAC7D,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,KAAK;SAChB,IAAI,EAAE;SACN,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,WAAW,EAAE,CAAC;IACnB,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACrF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,MAAM,CAAC,iBAAU,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,OAAO,CAAC;IACnB,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,OAAO,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,KAAK,EAAE,CAAC;QACR,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,KAAa;IAC9C,IAAI,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,KAAa;IACxC,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,iBAAU,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9C,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAgB,aAAa,CAAC,GAAW,EAAE,QAAuB;IAC9D,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,gBAAgB,CAAC,QAAQ,CAAC,IAAI,OAAO,EAAE,gBAAgB,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAClH,CAAC;IAED,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC9C,6FAA6F;IAC7F,yFAAyF;IACzF,MAAM,KAAK,GAAG,UAAU,IAAI,OAAO,CAAC;IAEpC,IAAI,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACtC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,QAAQ,GAAG,IAAI,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACrE,WAAW,GAAG,cAAc,CAAC;QAC7B,QAAQ,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI,gBAAgB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAE/C,8FAA8F;IAC9F,wFAAwF;IACxF,MAAM,kBAAkB,GAAG,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,MAAM,IAAI,CAAC,UAAU,CAAC;IAChF,IAAI,gBAAgB,GAAG,CAAC,IAAI,kBAAkB,IAAI,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,cAAc,EAAE,CAAC;QAC7G,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAClD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,KAAK,CAAC;YACb,WAAW,GAAG,cAAc,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;YAChB,gBAAgB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;AAC7D,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Header extraction helpers.
|
|
3
|
+
*
|
|
4
|
+
* Nothing in the pipeline may depend on a header being present. Every accessor returns null rather
|
|
5
|
+
* than throwing, and every decode is wrapped so that a malformed encoded word cannot abort a parse.
|
|
6
|
+
*/
|
|
7
|
+
import { type HeaderEntry, type MessageHeaders } from './model';
|
|
8
|
+
/** Decode MIME encoded words, falling back to the raw value on any failure. */
|
|
9
|
+
export declare function decodeWordsSafe(value: string): string;
|
|
10
|
+
/** Parse a structured header value such as Content-Type into a value plus parameters. */
|
|
11
|
+
export declare function parseHeaderValueSafe(value: string): {
|
|
12
|
+
value: string;
|
|
13
|
+
params: Record<string, string>;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Convert mailsplit's raw header lines into normalised entries.
|
|
17
|
+
*
|
|
18
|
+
* mailsplit returns `{ key, line }` where `line` still contains the field name and any folded
|
|
19
|
+
* continuations joined with CRLF. Lines with no colon at all are kept with an empty key so that
|
|
20
|
+
* malformed input is still visible to diagnostics.
|
|
21
|
+
*/
|
|
22
|
+
export declare function toHeaderEntries(list: readonly {
|
|
23
|
+
key: string;
|
|
24
|
+
line: string;
|
|
25
|
+
}[]): HeaderEntry[];
|
|
26
|
+
/** Strip angle brackets and surrounding whitespace from a Message-ID style value. */
|
|
27
|
+
export declare function normaliseMessageId(value: string | null): string | null;
|
|
28
|
+
/** Extract every angle bracketed id from a References style header. */
|
|
29
|
+
export declare function parseReferences(value: string | null): string[];
|
|
30
|
+
/** Collect the headers the analysis pipeline uses. All fields are optional by design. */
|
|
31
|
+
export declare function extractMessageHeaders(entries: readonly HeaderEntry[]): MessageHeaders;
|
|
32
|
+
//# sourceMappingURL=headers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/mime/headers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,SAAS,CAAC;AAEhE,+EAA+E;AAC/E,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAMrD;AAED,yFAAyF;AACzF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAOrG;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,SAAS;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,WAAW,EAAE,CAY7F;AAqBD,qFAAqF;AACrF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOtE;AAED,uEAAuE;AACvE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,EAAE,CAa9D;AAUD,yFAAyF;AACzF,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,cAAc,CAoBrF"}
|