@postalsys/email-content 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +288 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/analyze.d.ts +91 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +274 -0
- package/dist/analyze.js.map +1 -0
- package/dist/annotate/client.d.ts +64 -0
- package/dist/annotate/client.d.ts.map +1 -0
- package/dist/annotate/client.js +139 -0
- package/dist/annotate/client.js.map +1 -0
- package/dist/annotate/llm.d.ts +46 -0
- package/dist/annotate/llm.d.ts.map +1 -0
- package/dist/annotate/llm.js +107 -0
- package/dist/annotate/llm.js.map +1 -0
- package/dist/annotate/prompt.d.ts +38 -0
- package/dist/annotate/prompt.d.ts.map +1 -0
- package/dist/annotate/prompt.js +165 -0
- package/dist/annotate/prompt.js.map +1 -0
- package/dist/annotate/store.d.ts +43 -0
- package/dist/annotate/store.d.ts.map +1 -0
- package/dist/annotate/store.js +46 -0
- package/dist/annotate/store.js.map +1 -0
- package/dist/canonical/html.d.ts +33 -0
- package/dist/canonical/html.d.ts.map +1 -0
- package/dist/canonical/html.js +156 -0
- package/dist/canonical/html.js.map +1 -0
- package/dist/canonical/text.d.ts +75 -0
- package/dist/canonical/text.d.ts.map +1 -0
- package/dist/canonical/text.js +217 -0
- package/dist/canonical/text.js.map +1 -0
- package/dist/classify/confidence.d.ts +38 -0
- package/dist/classify/confidence.d.ts.map +1 -0
- package/dist/classify/confidence.js +59 -0
- package/dist/classify/confidence.js.map +1 -0
- package/dist/classify/reconcile.d.ts +52 -0
- package/dist/classify/reconcile.d.ts.map +1 -0
- package/dist/classify/reconcile.js +100 -0
- package/dist/classify/reconcile.js.map +1 -0
- package/dist/classify/rules.d.ts +51 -0
- package/dist/classify/rules.d.ts.map +1 -0
- package/dist/classify/rules.js +648 -0
- package/dist/classify/rules.js.map +1 -0
- package/dist/cli/annotate.d.ts +23 -0
- package/dist/cli/annotate.d.ts.map +1 -0
- package/dist/cli/annotate.js +328 -0
- package/dist/cli/annotate.js.map +1 -0
- package/dist/cli/args.d.ts +17 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +72 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/benchmark.d.ts +13 -0
- package/dist/cli/benchmark.d.ts.map +1 -0
- package/dist/cli/benchmark.js +172 -0
- package/dist/cli/benchmark.js.map +1 -0
- package/dist/cli/corpus.d.ts +9 -0
- package/dist/cli/corpus.d.ts.map +1 -0
- package/dist/cli/corpus.js +152 -0
- package/dist/cli/corpus.js.map +1 -0
- package/dist/cli/disagreements.d.ts +14 -0
- package/dist/cli/disagreements.d.ts.map +1 -0
- package/dist/cli/disagreements.js +133 -0
- package/dist/cli/disagreements.js.map +1 -0
- package/dist/cli/evaluate.d.ts +15 -0
- package/dist/cli/evaluate.d.ts.map +1 -0
- package/dist/cli/evaluate.js +215 -0
- package/dist/cli/evaluate.js.map +1 -0
- package/dist/cli/extract.d.ts +10 -0
- package/dist/cli/extract.d.ts.map +1 -0
- package/dist/cli/extract.js +139 -0
- package/dist/cli/extract.js.map +1 -0
- package/dist/cli/main.d.ts +9 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +147 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/corpus/analyze.d.ts +66 -0
- package/dist/corpus/analyze.d.ts.map +1 -0
- package/dist/corpus/analyze.js +214 -0
- package/dist/corpus/analyze.js.map +1 -0
- package/dist/corpus/migrations/index.d.ts +14 -0
- package/dist/corpus/migrations/index.d.ts.map +1 -0
- package/dist/corpus/migrations/index.js +158 -0
- package/dist/corpus/migrations/index.js.map +1 -0
- package/dist/corpus/sample.d.ts +66 -0
- package/dist/corpus/sample.d.ts.map +1 -0
- package/dist/corpus/sample.js +98 -0
- package/dist/corpus/sample.js.map +1 -0
- package/dist/corpus/scanner.d.ts +79 -0
- package/dist/corpus/scanner.d.ts.map +1 -0
- package/dist/corpus/scanner.js +256 -0
- package/dist/corpus/scanner.js.map +1 -0
- package/dist/corpus/stats.d.ts +76 -0
- package/dist/corpus/stats.d.ts.map +1 -0
- package/dist/corpus/stats.js +240 -0
- package/dist/corpus/stats.js.map +1 -0
- package/dist/corpus/store.d.ts +67 -0
- package/dist/corpus/store.d.ts.map +1 -0
- package/dist/corpus/store.js +196 -0
- package/dist/corpus/store.js.map +1 -0
- package/dist/corpus/thread-index.d.ts +31 -0
- package/dist/corpus/thread-index.d.ts.map +1 -0
- package/dist/corpus/thread-index.js +182 -0
- package/dist/corpus/thread-index.js.map +1 -0
- package/dist/corpus/worker.d.ts +21 -0
- package/dist/corpus/worker.d.ts.map +1 -0
- package/dist/corpus/worker.js +74 -0
- package/dist/corpus/worker.js.map +1 -0
- package/dist/detect/client.d.ts +28 -0
- package/dist/detect/client.d.ts.map +1 -0
- package/dist/detect/client.js +121 -0
- package/dist/detect/client.js.map +1 -0
- package/dist/detect/disclaimer.d.ts +12 -0
- package/dist/detect/disclaimer.d.ts.map +1 -0
- package/dist/detect/disclaimer.js +28 -0
- package/dist/detect/disclaimer.js.map +1 -0
- package/dist/detect/forward-header.d.ts +10 -0
- package/dist/detect/forward-header.d.ts.map +1 -0
- package/dist/detect/forward-header.js +35 -0
- package/dist/detect/forward-header.js.map +1 -0
- package/dist/detect/patterns/disclaimers.json +113 -0
- package/dist/detect/patterns/forward-headers.json +40 -0
- package/dist/detect/patterns/html-clients.json +157 -0
- package/dist/detect/patterns/quoted-headers.json +41 -0
- package/dist/detect/patterns/reply-headers.json +139 -0
- package/dist/detect/patterns/signatures.json +58 -0
- package/dist/detect/patterns.d.ts +43 -0
- package/dist/detect/patterns.d.ts.map +1 -0
- package/dist/detect/patterns.js +125 -0
- package/dist/detect/patterns.js.map +1 -0
- package/dist/detect/reply-header.d.ts +44 -0
- package/dist/detect/reply-header.d.ts.map +1 -0
- package/dist/detect/reply-header.js +134 -0
- package/dist/detect/reply-header.js.map +1 -0
- package/dist/detect/signature.d.ts +28 -0
- package/dist/detect/signature.d.ts.map +1 -0
- package/dist/detect/signature.js +77 -0
- package/dist/detect/signature.js.map +1 -0
- package/dist/detect/traffic.d.ts +26 -0
- package/dist/detect/traffic.d.ts.map +1 -0
- package/dist/detect/traffic.js +98 -0
- package/dist/detect/traffic.js.map +1 -0
- package/dist/eval/metrics.d.ts +87 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +185 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/index.d.ts +54 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +163 -0
- package/dist/index.js.map +1 -0
- package/dist/match/history.d.ts +62 -0
- package/dist/match/history.d.ts.map +1 -0
- package/dist/match/history.js +128 -0
- package/dist/match/history.js.map +1 -0
- package/dist/match/normalize.d.ts +33 -0
- package/dist/match/normalize.d.ts.map +1 -0
- package/dist/match/normalize.js +63 -0
- package/dist/match/normalize.js.map +1 -0
- package/dist/match/shingle.d.ts +44 -0
- package/dist/match/shingle.d.ts.map +1 -0
- package/dist/match/shingle.js +78 -0
- package/dist/match/shingle.js.map +1 -0
- package/dist/mime/charset.d.ts +33 -0
- package/dist/mime/charset.d.ts.map +1 -0
- package/dist/mime/charset.js +118 -0
- package/dist/mime/charset.js.map +1 -0
- package/dist/mime/headers.d.ts +32 -0
- package/dist/mime/headers.d.ts.map +1 -0
- package/dist/mime/headers.js +127 -0
- package/dist/mime/headers.js.map +1 -0
- package/dist/mime/model.d.ts +111 -0
- package/dist/mime/model.d.ts.map +1 -0
- package/dist/mime/model.js +17 -0
- package/dist/mime/model.js.map +1 -0
- package/dist/mime/parse.d.ts +27 -0
- package/dist/mime/parse.d.ts.map +1 -0
- package/dist/mime/parse.js +343 -0
- package/dist/mime/parse.js.map +1 -0
- package/dist/segment/features.d.ts +29 -0
- package/dist/segment/features.d.ts.map +1 -0
- package/dist/segment/features.js +102 -0
- package/dist/segment/features.js.map +1 -0
- package/dist/segment/html-blocks.d.ts +28 -0
- package/dist/segment/html-blocks.d.ts.map +1 -0
- package/dist/segment/html-blocks.js +596 -0
- package/dist/segment/html-blocks.js.map +1 -0
- package/dist/segment/text-blocks.d.ts +30 -0
- package/dist/segment/text-blocks.d.ts.map +1 -0
- package/dist/segment/text-blocks.js +204 -0
- package/dist/segment/text-blocks.js.map +1 -0
- package/dist/segment/types.d.ts +88 -0
- package/dist/segment/types.d.ts.map +1 -0
- package/dist/segment/types.js +11 -0
- package/dist/segment/types.js.map +1 -0
- package/dist/types.d.ts +85 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +17 -0
- package/dist/types.js.map +1 -0
- package/dist/util/hash.d.ts +49 -0
- package/dist/util/hash.d.ts.map +1 -0
- package/dist/util/hash.js +0 -0
- package/dist/util/hash.js.map +1 -0
- package/dist/util/lang.d.ts +25 -0
- package/dist/util/lang.d.ts.map +1 -0
- package/dist/util/lang.js +135 -0
- package/dist/util/lang.js.map +1 -0
- package/dist/util/worker-pool.d.ts +51 -0
- package/dist/util/worker-pool.d.ts.map +1 -0
- package/dist/util/worker-pool.js +155 -0
- package/dist/util/worker-pool.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1,648 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Stage 6: rule based sequence classification.
|
|
4
|
+
*
|
|
5
|
+
* Blocks are classified in context, never independently. The rules encode the four sequences that
|
|
6
|
+
* matter, in the order they are resolved:
|
|
7
|
+
*
|
|
8
|
+
* NEW_CONTENT, SIGNATURE, REPLY_HEADER, REPLY_HISTORY top posting, the common case
|
|
9
|
+
* INLINE_QUOTE, NEW_CONTENT, INLINE_QUOTE, NEW_CONTENT interleaved answers
|
|
10
|
+
* NEW_CONTENT, FORWARD_HEADER, FORWARDED_CONTENT forwarding with a comment
|
|
11
|
+
* REPLY_HISTORY, NEW_CONTENT bottom posting
|
|
12
|
+
*
|
|
13
|
+
* The load bearing rule is the anchor requirement. A quote region may only become REPLY_HISTORY
|
|
14
|
+
* when something independent of the quoting itself says that it is history: an attribution line, a
|
|
15
|
+
* client quote container, or a match against a message that already existed. Quote depth alone
|
|
16
|
+
* gives INLINE_QUOTE or UNKNOWN, both of which stay visible.
|
|
17
|
+
*
|
|
18
|
+
* The reason a bare quote is not treated as history is the difference between the two examples in
|
|
19
|
+
* the brief. Quoted questions with answers under them are context the reader needs. A trailing
|
|
20
|
+
* quoted copy of the previous mail is noise. Only the surrounding structure tells them apart.
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.classifyBlocks = classifyBlocks;
|
|
24
|
+
const confidence_1 = require("./confidence");
|
|
25
|
+
/**
|
|
26
|
+
* Evidence ids that mark the first line of a signature rather than its body.
|
|
27
|
+
*
|
|
28
|
+
* A closing greeting or a "sent from my phone" line opens a signature. A phone number or a street
|
|
29
|
+
* address sits inside one.
|
|
30
|
+
*/
|
|
31
|
+
const TOP_OF_SIGNATURE = new Set(['signature:best_regards', 'signature:sent_from_device', 'signature:sent_from_localised', 'signature:dash_dash_space']);
|
|
32
|
+
/** The signature evidence id on a block, if any. */
|
|
33
|
+
function topSignal(block) {
|
|
34
|
+
for (const evidence of block.evidence) {
|
|
35
|
+
if (evidence.kind === 'signature' && TOP_OF_SIGNATURE.has(evidence.id)) {
|
|
36
|
+
return evidence.id;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return '';
|
|
40
|
+
}
|
|
41
|
+
/** Highest strength evidence of a given kind attached to a block. */
|
|
42
|
+
function strengthOf(block, kinds) {
|
|
43
|
+
let best = 0;
|
|
44
|
+
for (const evidence of block.evidence) {
|
|
45
|
+
if (kinds.includes(evidence.kind) && evidence.strength > best) {
|
|
46
|
+
best = evidence.strength;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return best;
|
|
50
|
+
}
|
|
51
|
+
function hasWords(block) {
|
|
52
|
+
return block.features.wordCount > 0 && !block.features.isAllBlank;
|
|
53
|
+
}
|
|
54
|
+
/** A quoted block, in either representation. */
|
|
55
|
+
function isQuote(block) {
|
|
56
|
+
return block.kind === 'quote' || block.features.quoteDepth > 0 || block.features.blockquoteDepth > 0;
|
|
57
|
+
}
|
|
58
|
+
/** Client generated boilerplate that introduces a quote. Not authored by the sender. */
|
|
59
|
+
function isAnchorBlock(block) {
|
|
60
|
+
return block.kind === 'reply_header' || block.kind === 'quoted_header_block';
|
|
61
|
+
}
|
|
62
|
+
/** Group blocks into alternating quoted and unquoted runs, ignoring blank blocks. */
|
|
63
|
+
function findRuns(blocks) {
|
|
64
|
+
const runs = [];
|
|
65
|
+
for (const block of blocks) {
|
|
66
|
+
if (block.features.isAllBlank) {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const quoted = isQuote(block) || isAnchorBlock(block);
|
|
70
|
+
const last = runs[runs.length - 1];
|
|
71
|
+
if (last && last.quoted === quoted) {
|
|
72
|
+
last.end = block.index;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
runs.push({ start: block.index, end: block.index, quoted });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return runs;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Classify one representation's blocks.
|
|
82
|
+
*
|
|
83
|
+
* Runs in four passes so that later passes can see the whole picture: footers first, because they
|
|
84
|
+
* bound where history can start; then anchors; then regions; then defaults.
|
|
85
|
+
*/
|
|
86
|
+
function classifyBlocks(blocks, options = {}) {
|
|
87
|
+
const config = options.config ?? confidence_1.DEFAULT_CONFIDENCE;
|
|
88
|
+
const matched = options.historyMatchedIndices ?? new Set();
|
|
89
|
+
const senderTokens = options.senderTokens ?? [];
|
|
90
|
+
const decisions = blocks.map(block => ({
|
|
91
|
+
block,
|
|
92
|
+
label: 'NEW_CONTENT',
|
|
93
|
+
confidence: 0.95,
|
|
94
|
+
reasons: ['new_content_default']
|
|
95
|
+
}));
|
|
96
|
+
const set = (index, label, confidence, reasons) => {
|
|
97
|
+
const decision = decisions[index];
|
|
98
|
+
if (decision) {
|
|
99
|
+
decision.label = label;
|
|
100
|
+
decision.confidence = confidence;
|
|
101
|
+
decision.reasons = reasons;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
// Pass 1: locate anchors.
|
|
105
|
+
let replyAnchor = -1;
|
|
106
|
+
let replyAnchorStrength = 0;
|
|
107
|
+
let forwardAnchor = -1;
|
|
108
|
+
let forwardAnchorStrength = 0;
|
|
109
|
+
for (const block of blocks) {
|
|
110
|
+
const forward = strengthOf(block, ['forward-anchor']);
|
|
111
|
+
if (forward > 0 && forwardAnchor === -1) {
|
|
112
|
+
forwardAnchor = block.index;
|
|
113
|
+
forwardAnchorStrength = forward;
|
|
114
|
+
}
|
|
115
|
+
const reply = strengthOf(block, ['reply-anchor']);
|
|
116
|
+
if (reply > 0 && replyAnchor === -1) {
|
|
117
|
+
replyAnchor = block.index;
|
|
118
|
+
replyAnchorStrength = reply;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// A quote container in HTML is an anchor in its own right, but only when something corroborates
|
|
122
|
+
// it.
|
|
123
|
+
//
|
|
124
|
+
// A client that quotes a message emits an attribution line with the container, or nests the
|
|
125
|
+
// quoted text in a blockquote, or both. A container with neither is not reliably a quote
|
|
126
|
+
// boundary: mailing list relays and some composers wrap arbitrary spans in a gmail_quote div,
|
|
127
|
+
// and one message in the working corpus splits a single sentence across the container boundary.
|
|
128
|
+
// Trusting that container would hide half a sentence of genuine content, so an uncorroborated
|
|
129
|
+
// container is left as ordinary evidence and cannot anchor a hide.
|
|
130
|
+
const hasAttribution = blocks.some(block => isAnchorBlock(block) || block.evidence.some(item => item.kind === 'attribution' || item.kind === 'reply-anchor'));
|
|
131
|
+
const hasBlockquote = blocks.some(block => block.features.blockquoteDepth > 0);
|
|
132
|
+
let containerAnchor = -1;
|
|
133
|
+
let containerStrength = 0;
|
|
134
|
+
if (hasAttribution || hasBlockquote) {
|
|
135
|
+
for (const block of blocks) {
|
|
136
|
+
const container = strengthOf(block, ['quote-container']);
|
|
137
|
+
if (container >= config.minAnchorStrength && containerAnchor === -1) {
|
|
138
|
+
containerAnchor = block.index;
|
|
139
|
+
containerStrength = container;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (containerAnchor !== -1 && (replyAnchor === -1 || containerAnchor < replyAnchor)) {
|
|
144
|
+
replyAnchor = containerAnchor;
|
|
145
|
+
replyAnchorStrength = Math.max(replyAnchorStrength, containerStrength);
|
|
146
|
+
}
|
|
147
|
+
// Some clients mark the boundary rather than wrapping the quote: Outlook Web emits an empty
|
|
148
|
+
// #divRplyFwdMsg, Zimbra an #zwchr rule, Outlook desktop a bordered element. Everything after
|
|
149
|
+
// such a separator is history, even though none of it is inside a blockquote.
|
|
150
|
+
let separatorAnchor = -1;
|
|
151
|
+
let separatorStrength = 0;
|
|
152
|
+
for (const block of blocks) {
|
|
153
|
+
const strength = strengthOf(block, ['separator']);
|
|
154
|
+
if (strength >= config.minAnchorStrength) {
|
|
155
|
+
separatorAnchor = block.index;
|
|
156
|
+
separatorStrength = strength;
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
// The plain text underscore rule is far too common to trust on its own. It only counts when an
|
|
161
|
+
// embedded header block follows it closely, which is the shape Outlook actually produces.
|
|
162
|
+
if (separatorAnchor === -1) {
|
|
163
|
+
for (const block of blocks) {
|
|
164
|
+
if (block.kind !== 'separator' || strengthOf(block, ['separator']) < 0.6) {
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
let seen = 0;
|
|
168
|
+
for (let i = block.index + 1; i < blocks.length && seen < 3; i++) {
|
|
169
|
+
const candidate = blocks[i];
|
|
170
|
+
if (candidate.features.isAllBlank) {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
seen++;
|
|
174
|
+
if (candidate.kind === 'quoted_header_block') {
|
|
175
|
+
separatorAnchor = block.index;
|
|
176
|
+
separatorStrength = 0.9;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (separatorAnchor !== -1) {
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Pass 2: trailing footers, searched only within the region the sender actually authored.
|
|
186
|
+
//
|
|
187
|
+
// The footer scan has to know where history starts, which is why it runs after anchors. In a
|
|
188
|
+
// top posted reply the signature sits between the new text and the quote, so a naive scan
|
|
189
|
+
// backwards from the end of the message would hit the quoted history first and give up before
|
|
190
|
+
// ever reaching the delimiter. Anything after the first anchor belongs to the quoted message
|
|
191
|
+
// and is classified by the region rules instead.
|
|
192
|
+
const firstQuoted = blocks.find(block => isQuote(block) && !block.features.isAllBlank)?.index ?? blocks.length;
|
|
193
|
+
const authoredEnd = Math.min(firstQuoted, replyAnchor === -1 ? blocks.length : replyAnchor, separatorAnchor === -1 ? blocks.length : separatorAnchor, forwardAnchor === -1 ? blocks.length : forwardAnchor);
|
|
194
|
+
// An RFC 3676 delimiter is the one high confidence signature signal. Everything from it to the
|
|
195
|
+
// end of the authored region is the signature.
|
|
196
|
+
for (let i = authoredEnd - 1; i >= 0; i--) {
|
|
197
|
+
const block = blocks[i];
|
|
198
|
+
if (isQuote(block)) {
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
// Match on the delimiter evidence, not on the block kind. In plain text a "-- " line is a
|
|
202
|
+
// block of its own and comes out as kind 'separator', but in HTML it is the first line of a
|
|
203
|
+
// paragraph that also holds the signature body. Requiring 'separator' meant the one high
|
|
204
|
+
// confidence signature signal never fired on HTML at all.
|
|
205
|
+
if (block.evidence.some(item => item.id === 'signature:dash_dash_space' && item.strength >= 0.9)) {
|
|
206
|
+
for (let j = i; j < authoredEnd; j++) {
|
|
207
|
+
set(j, 'SIGNATURE', 0.95, ['dash_dash_delimiter']);
|
|
208
|
+
}
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Trailing disclaimers, anywhere in the message.
|
|
213
|
+
//
|
|
214
|
+
// A gateway appends its legal footer to the outgoing message, so it lands after the quoted
|
|
215
|
+
// history, outside the authored region the footer scan above covers. Measured against the
|
|
216
|
+
// reference set, disclaimers sit at a mean relative position of 0.86 and 81 percent of them are
|
|
217
|
+
// in the final quarter of the message, so bounding the scan to the authored region missed
|
|
218
|
+
// essentially all of them.
|
|
219
|
+
//
|
|
220
|
+
// Only blocks still considered authored are converted, so this cannot pull anything out of a
|
|
221
|
+
// history or forwarded region, and DISCLAIMER never hides, it only collapses.
|
|
222
|
+
for (const block of blocks) {
|
|
223
|
+
const decision = decisions[block.index];
|
|
224
|
+
if (!decision || decision.label !== 'NEW_CONTENT') {
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
const strength = strengthOf(block, ['disclaimer']);
|
|
228
|
+
if (strength >= 0.7) {
|
|
229
|
+
decision.label = 'DISCLAIMER';
|
|
230
|
+
decision.confidence = Math.min(0.95, strength);
|
|
231
|
+
decision.reasons = ['disclaimer_phrase'];
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
// Then the signature region.
|
|
235
|
+
//
|
|
236
|
+
// A signature is not one block. In HTML it is typically a run of short blocks at the end of the
|
|
237
|
+
// authored region, one line each: name, then title, then company, then phone, then links. Only
|
|
238
|
+
// one of those carries anything a per block rule would recognise, which is why checking the
|
|
239
|
+
// last block or two found almost nothing.
|
|
240
|
+
//
|
|
241
|
+
// So walk back from the end of the authored region collecting short blocks, stop at the first
|
|
242
|
+
// real paragraph or quote, and accept the run if anything in it looks like a sign-off: contact
|
|
243
|
+
// details, or a closing greeting, or a device phrase. The caps keep an unsigned message from
|
|
244
|
+
// swallowing its own last paragraphs, and SIGNATURE only ever collapses, so the cost of being
|
|
245
|
+
// wrong here is bounded.
|
|
246
|
+
const MAX_SIGNATURE_BLOCKS = 6;
|
|
247
|
+
const MAX_SIGNATURE_WORDS = 60;
|
|
248
|
+
const MAX_BLOCK_WORDS = 25;
|
|
249
|
+
let signatureStart = -1;
|
|
250
|
+
let hasContact = false;
|
|
251
|
+
let hasPhrase = false;
|
|
252
|
+
let hasSenderName = false;
|
|
253
|
+
let phraseStrength = 0;
|
|
254
|
+
let words = 0;
|
|
255
|
+
let collected = 0;
|
|
256
|
+
for (let i = authoredEnd - 1; i >= 0 && collected < MAX_SIGNATURE_BLOCKS; i--) {
|
|
257
|
+
const block = blocks[i];
|
|
258
|
+
const decision = decisions[i];
|
|
259
|
+
if (!decision) {
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
if (block.features.isAllBlank) {
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (decision.label === 'SIGNATURE') {
|
|
266
|
+
signatureStart = i;
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
if (isQuote(block) || isAnchorBlock(block) || decision.label === 'DISCLAIMER') {
|
|
270
|
+
break;
|
|
271
|
+
}
|
|
272
|
+
if (block.features.wordCount > MAX_BLOCK_WORDS) {
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
words += block.features.wordCount;
|
|
276
|
+
if (words > MAX_SIGNATURE_WORDS) {
|
|
277
|
+
break;
|
|
278
|
+
}
|
|
279
|
+
if (block.features.hasEmailAddress || block.features.hasPhoneNumber || block.features.hasUrl) {
|
|
280
|
+
hasContact = true;
|
|
281
|
+
}
|
|
282
|
+
const strength = strengthOf(block, ['signature']);
|
|
283
|
+
if (strength > 0) {
|
|
284
|
+
hasPhrase = true;
|
|
285
|
+
phraseStrength = Math.max(phraseStrength, strength);
|
|
286
|
+
}
|
|
287
|
+
if (senderTokens.length && !hasSenderName) {
|
|
288
|
+
const lowered = block.content.toLowerCase();
|
|
289
|
+
if (senderTokens.some(token => lowered.includes(token))) {
|
|
290
|
+
hasSenderName = true;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
signatureStart = i;
|
|
294
|
+
collected++;
|
|
295
|
+
// A closing greeting or a device phrase is the top of a signature, not part of the body
|
|
296
|
+
// above it. Stop here, otherwise the scan keeps walking back over short sentences and
|
|
297
|
+
// eventually swallows the message it was supposed to be signing.
|
|
298
|
+
//
|
|
299
|
+
// Only those two count. Contact lines, postal addresses and the generic shape heuristic are
|
|
300
|
+
// signature *body*, and treating them as the top stopped the scan on the first line it saw,
|
|
301
|
+
// which is why a six line corporate signature was recognised as no signature at all.
|
|
302
|
+
if (TOP_OF_SIGNATURE.has(topSignal(block))) {
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
// Two independent signals, or one very strong one. Requiring only "something looks like a
|
|
307
|
+
// sign-off" labelled a quarter of the corpus's closing paragraphs as signature: precision was
|
|
308
|
+
// 24 percent and it started collapsing real content.
|
|
309
|
+
const strongPhrase = phraseStrength >= 0.9;
|
|
310
|
+
// A signature signs something. If the candidate run starts at the first authored block there is
|
|
311
|
+
// nothing above it to sign, and the message is a short note that happens to end in contact
|
|
312
|
+
// details, not a signature block. Without this guard the region swallowed whole short messages
|
|
313
|
+
// and left only a collapsed footer visible, which the corpus checks caught on 42 of 3,000
|
|
314
|
+
// messages.
|
|
315
|
+
let hasContentAbove = false;
|
|
316
|
+
for (let i = 0; i < signatureStart; i++) {
|
|
317
|
+
const block = blocks[i];
|
|
318
|
+
const decision = decisions[i];
|
|
319
|
+
if (block && decision && !block.features.isAllBlank && decision.label === 'NEW_CONTENT' && hasWords(block)) {
|
|
320
|
+
hasContentAbove = true;
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
// The sender's own name counts alongside contact details and sign-off phrases. Requiring two
|
|
325
|
+
// signals still holds, so a message that merely mentions the sender's name in passing does not
|
|
326
|
+
// become a signature.
|
|
327
|
+
const signals = [hasContact, hasPhrase, hasSenderName].filter(Boolean).length;
|
|
328
|
+
const accepted = hasContentAbove && (strongPhrase || signals >= 2);
|
|
329
|
+
if (signatureStart !== -1 && accepted) {
|
|
330
|
+
const confidence = phraseStrength >= 0.9 ? 0.95 : signals >= 3 ? 0.9 : signals >= 2 ? 0.85 : 0.76;
|
|
331
|
+
const reason = phraseStrength >= 0.9 ? 'mobile_signature_phrase' : 'signature_position_and_length';
|
|
332
|
+
for (let i = signatureStart; i < authoredEnd; i++) {
|
|
333
|
+
const decision = decisions[i];
|
|
334
|
+
const block = blocks[i];
|
|
335
|
+
if (!decision || !block || decision.label === 'DISCLAIMER') {
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
if (isQuote(block) || isAnchorBlock(block)) {
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
set(i, 'SIGNATURE', confidence, [reason]);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// Pass 3: regions.
|
|
345
|
+
const runs = findRuns(blocks);
|
|
346
|
+
const quotedRuns = runs.filter(run => run.quoted);
|
|
347
|
+
const contentRunsAfterFirstQuote = runs.filter(run => !run.quoted && quotedRuns.length > 0 && run.start > quotedRuns[0].start);
|
|
348
|
+
// Interleaved: more than one quoted run, with authored content between them. Every quoted run
|
|
349
|
+
// is then context for the answer beside it, not disposable history.
|
|
350
|
+
const interleaved = quotedRuns.length > 1 &&
|
|
351
|
+
contentRunsAfterFirstQuote.some(run => {
|
|
352
|
+
for (let i = run.start; i <= run.end; i++) {
|
|
353
|
+
const decision = decisions[i];
|
|
354
|
+
const block = blocks[i];
|
|
355
|
+
if (!decision || !block) {
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
if (decision.label === 'NEW_CONTENT' && hasWords(block) && !isAnchorBlock(block)) {
|
|
359
|
+
return true;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return false;
|
|
363
|
+
});
|
|
364
|
+
if (forwardAnchor !== -1) {
|
|
365
|
+
// Everything from the forward marker to the end is forwarded material. It is collapsed and
|
|
366
|
+
// kept, never hidden: the sender included it deliberately.
|
|
367
|
+
for (let i = forwardAnchor; i < blocks.length; i++) {
|
|
368
|
+
const decision = decisions[i];
|
|
369
|
+
if (!decision || decision.label === 'DISCLAIMER') {
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
set(i, 'FORWARDED_CONTENT', Math.min(0.95, forwardAnchorStrength), ['preceded_by_forward_header']);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
for (const run of quotedRuns) {
|
|
376
|
+
if (forwardAnchor !== -1 && run.start >= forwardAnchor) {
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
const anchored = replyAnchor !== -1 && replyAnchor <= run.end;
|
|
380
|
+
const anchorStrength = anchored ? replyAnchorStrength : 0;
|
|
381
|
+
// Is there authored content after this run that is not a footer?
|
|
382
|
+
let newContentAfter = false;
|
|
383
|
+
for (let i = run.end + 1; i < blocks.length; i++) {
|
|
384
|
+
const decision = decisions[i];
|
|
385
|
+
const block = blocks[i];
|
|
386
|
+
if (!decision || !block || block.features.isAllBlank) {
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
if (decision.label === 'SIGNATURE' || decision.label === 'DISCLAIMER') {
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (isQuote(block) || isAnchorBlock(block)) {
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
if (hasWords(block)) {
|
|
396
|
+
newContentAfter = true;
|
|
397
|
+
break;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
let label;
|
|
401
|
+
let confidence;
|
|
402
|
+
const reasons = [];
|
|
403
|
+
// Deepest quoting in this run. Depth 1 is the message being replied to; depth 2 or more is
|
|
404
|
+
// a quote inside a quote, which someone else already quoted before this conversation
|
|
405
|
+
// reached the sender.
|
|
406
|
+
let runDepth = 0;
|
|
407
|
+
for (let i = run.start; i <= run.end; i++) {
|
|
408
|
+
const block = blocks[i];
|
|
409
|
+
if (block) {
|
|
410
|
+
runDepth = Math.max(runDepth, block.features.quoteDepth, block.features.blockquoteDepth);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
// Inline quoting is answering someone point by point, and it happens at depth 1: you quote
|
|
414
|
+
// the person you are replying to and write underneath. Nested quoting is a chain, and a run
|
|
415
|
+
// with an answer somewhere after it elsewhere in the message says nothing about whether this
|
|
416
|
+
// particular nested run is context. Measured on the reference set, 220 of the blocks wrongly
|
|
417
|
+
// called inline quote were at depth 2 or deeper.
|
|
418
|
+
const eligibleForInline = runDepth <= 1;
|
|
419
|
+
if (interleaved && eligibleForInline && newContentAfter) {
|
|
420
|
+
// Answered here, so it is the context for that answer.
|
|
421
|
+
label = 'INLINE_QUOTE';
|
|
422
|
+
confidence = 0.9;
|
|
423
|
+
reasons.push('interleaved_with_new_content', 'new_content_after_region');
|
|
424
|
+
}
|
|
425
|
+
else if (anchored) {
|
|
426
|
+
label = 'REPLY_HISTORY';
|
|
427
|
+
if (anchored) {
|
|
428
|
+
reasons.push(replyAnchor === containerAnchor ? 'inside_quote_container' : 'preceded_by_reply_header');
|
|
429
|
+
}
|
|
430
|
+
if (newContentAfter) {
|
|
431
|
+
// Bottom posting. The quote is still history, but the reader is depending on it for
|
|
432
|
+
// context more often than in the top posted case, so it is collapsed rather than
|
|
433
|
+
// hidden unless history matching confirmed it.
|
|
434
|
+
reasons.push('new_content_after_region');
|
|
435
|
+
confidence = 0.9;
|
|
436
|
+
}
|
|
437
|
+
else {
|
|
438
|
+
reasons.push('final_contiguous_quote_region', 'no_new_content_after_region');
|
|
439
|
+
confidence = Math.min(0.97, 0.6 + anchorStrength * 0.4);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
else if (newContentAfter && eligibleForInline) {
|
|
443
|
+
// Quoted text with an answer under it and nothing saying it is history. This is the
|
|
444
|
+
// inline question and answer shape, and the quote is the context for the answer.
|
|
445
|
+
label = 'INLINE_QUOTE';
|
|
446
|
+
confidence = 0.9;
|
|
447
|
+
reasons.push('new_content_after_region', 'no_anchor_for_quote');
|
|
448
|
+
}
|
|
449
|
+
else {
|
|
450
|
+
// Quoted text at the end of a message with no anchor at all. It looks like history but
|
|
451
|
+
// nothing confirms it, so it stays visible.
|
|
452
|
+
label = 'UNKNOWN';
|
|
453
|
+
confidence = 0.5;
|
|
454
|
+
reasons.push('no_anchor_for_quote', 'ambiguous_kept_visible');
|
|
455
|
+
}
|
|
456
|
+
for (let i = run.start; i <= run.end; i++) {
|
|
457
|
+
const decision = decisions[i];
|
|
458
|
+
const block = blocks[i];
|
|
459
|
+
if (!decision || !block) {
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
if (decision.label === 'SIGNATURE' || decision.label === 'DISCLAIMER') {
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
// A generic blockquote alone is weaker evidence than a client specific container.
|
|
466
|
+
const weakContainer = block.evidence.some(item => item.id === 'html:generic_blockquote') && strengthOf(block, ['quote-container']) < config.minAnchorStrength;
|
|
467
|
+
const adjusted = label === 'REPLY_HISTORY' && weakContainer && !anchored ? Math.min(confidence, 0.8) : confidence;
|
|
468
|
+
set(i, label, adjusted, [...reasons]);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
// Unmarked history after a strong anchor.
|
|
472
|
+
//
|
|
473
|
+
// Outlook and several other clients embed the previous message as plain paragraphs with no
|
|
474
|
+
// quote prefix at all, introduced by "-----Original Message-----" and a From/Sent/To/Subject
|
|
475
|
+
// block. Nothing in that text is marked as quoted, so the run rules above never see it and the
|
|
476
|
+
// whole previous message stays visible as new content.
|
|
477
|
+
//
|
|
478
|
+
// The sweep is bounded deliberately: it runs from the anchor only as far as the first quoted
|
|
479
|
+
// block. That is what keeps a bottom posted reply safe. In "On X wrote: / > quote / my answer",
|
|
480
|
+
// the anchor is immediately followed by a quoted block, so the sweep covers the anchor alone
|
|
481
|
+
// and the answer below is untouched.
|
|
482
|
+
if (replyAnchor !== -1 && replyAnchorStrength >= config.minAnchorStrength && !interleaved) {
|
|
483
|
+
let limit = blocks.length;
|
|
484
|
+
for (let i = replyAnchor + 1; i < blocks.length; i++) {
|
|
485
|
+
const block = blocks[i];
|
|
486
|
+
if (isQuote(block)) {
|
|
487
|
+
limit = i;
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
// Corroboration: Outlook's shape is the separator followed closely by the embedded header
|
|
492
|
+
// block. With it, this is safe to hide. Without it, the region is collapsed but stays
|
|
493
|
+
// recoverable, because a lone anchor could in principle be text the sender typed.
|
|
494
|
+
let corroborated = false;
|
|
495
|
+
let seen = 0;
|
|
496
|
+
for (let i = replyAnchor; i < blocks.length && seen < 4; i++) {
|
|
497
|
+
const block = blocks[i];
|
|
498
|
+
if (block.features.isAllBlank) {
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
seen++;
|
|
502
|
+
if (block.kind === 'quoted_header_block' || block.features.headerFieldLines >= 2) {
|
|
503
|
+
corroborated = true;
|
|
504
|
+
break;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
const reasons = corroborated
|
|
508
|
+
? ['preceded_by_reply_header', 'contains_old_message_headers', 'no_new_content_after_region']
|
|
509
|
+
: ['preceded_by_reply_header', 'ambiguous_kept_visible'];
|
|
510
|
+
const confidence = corroborated ? 0.96 : 0.9;
|
|
511
|
+
for (let i = replyAnchor; i < limit; i++) {
|
|
512
|
+
const decision = decisions[i];
|
|
513
|
+
if (!decision) {
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
if (decision.label === 'FORWARDED_CONTENT' || decision.label === 'INLINE_QUOTE') {
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
// The attribution line was already classified by the run rules, which could not see the
|
|
520
|
+
// unmarked history that follows it. Raising it to the sweep's confidence keeps the whole
|
|
521
|
+
// region as one segment instead of splitting it at the anchor.
|
|
522
|
+
if (decision.label === 'REPLY_HISTORY' && decision.confidence >= confidence) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
decision.label = 'REPLY_HISTORY';
|
|
526
|
+
decision.confidence = confidence;
|
|
527
|
+
decision.reasons = [...reasons];
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
// A client reply separator makes everything after it history, whether or not it is quoted.
|
|
531
|
+
// Footers are left alone: a signature after the separator belongs to the quoted message and is
|
|
532
|
+
// already inside the region, while one before it belongs to the sender.
|
|
533
|
+
if (separatorAnchor !== -1 && !interleaved) {
|
|
534
|
+
const reasons = ['outlook_reply_separator', 'final_contiguous_quote_region'];
|
|
535
|
+
for (let i = separatorAnchor; i < blocks.length; i++) {
|
|
536
|
+
const decision = decisions[i];
|
|
537
|
+
const block = blocks[i];
|
|
538
|
+
if (!decision || !block) {
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
if (decision.label === 'FORWARDED_CONTENT' || decision.label === 'REPLY_HISTORY' || decision.label === 'INLINE_QUOTE') {
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
decision.label = 'REPLY_HISTORY';
|
|
545
|
+
decision.confidence = Math.min(0.96, 0.6 + separatorStrength * 0.4);
|
|
546
|
+
decision.reasons = [...reasons];
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
// An anchor line that sits outside any quoted run still belongs to the region it introduces.
|
|
550
|
+
for (const block of blocks) {
|
|
551
|
+
if (!isAnchorBlock(block)) {
|
|
552
|
+
continue;
|
|
553
|
+
}
|
|
554
|
+
const decision = decisions[block.index];
|
|
555
|
+
if (!decision || decision.label !== 'NEW_CONTENT') {
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
const next = decisions[block.index + 1] ?? decisions[block.index + 2];
|
|
559
|
+
if (next && (next.label === 'REPLY_HISTORY' || next.label === 'INLINE_QUOTE' || next.label === 'FORWARDED_CONTENT')) {
|
|
560
|
+
set(block.index, next.label, next.confidence, [...next.reasons]);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
// Stage 5 evidence, applied strictly per block.
|
|
564
|
+
//
|
|
565
|
+
// A match proves provenance: this exact text existed before the message was written. It does
|
|
566
|
+
// not prove function. So a match may confirm a region the structure already read as history,
|
|
567
|
+
// and may rescue a quote that had no anchor at all, but it may never override a functional
|
|
568
|
+
// judgment:
|
|
569
|
+
//
|
|
570
|
+
// - INLINE_QUOTE is never promoted. Every inline quote appears in the parent by definition,
|
|
571
|
+
// so matching would hide exactly the context an answer depends on.
|
|
572
|
+
// - NEW_CONTENT is never promoted. A sender restating something from the parent is still
|
|
573
|
+
// writing it now.
|
|
574
|
+
//
|
|
575
|
+
// Applying this per block rather than per run matters. An earlier version promoted a whole run
|
|
576
|
+
// when any block in it matched, which hid authored text sitting beside quoted text and produced
|
|
577
|
+
// the only catastrophic failures measured on the reference set.
|
|
578
|
+
for (const index of matched) {
|
|
579
|
+
const decision = decisions[index];
|
|
580
|
+
const block = blocks[index];
|
|
581
|
+
if (!decision || !block) {
|
|
582
|
+
continue;
|
|
583
|
+
}
|
|
584
|
+
if (decision.label === 'REPLY_HISTORY') {
|
|
585
|
+
// Provenance alone does not license hiding. A quoted block with the sender's answer
|
|
586
|
+
// underneath is still the context for that answer, and a forwarded message is still
|
|
587
|
+
// something the sender chose to pass on, however certainly it existed before. Both stay
|
|
588
|
+
// collapsed, so a match confirms without ever crossing the hide threshold on its own.
|
|
589
|
+
const trailingRegion = decision.reasons.includes('no_new_content_after_region');
|
|
590
|
+
const insideForward = forwardAnchor !== -1 && index >= forwardAnchor;
|
|
591
|
+
const ceiling = trailingRegion && !insideForward ? 0.98 : 0.94;
|
|
592
|
+
decision.confidence = Math.max(decision.confidence, ceiling);
|
|
593
|
+
if (!decision.reasons.includes('matched_parent_message')) {
|
|
594
|
+
decision.reasons.push('matched_parent_message');
|
|
595
|
+
}
|
|
596
|
+
continue;
|
|
597
|
+
}
|
|
598
|
+
if (decision.label === 'UNKNOWN' && (isQuote(block) || isAnchorBlock(block))) {
|
|
599
|
+
decision.label = 'REPLY_HISTORY';
|
|
600
|
+
decision.confidence = 0.96;
|
|
601
|
+
decision.reasons = ['matched_parent_message', 'quote_depth_positive'];
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
// Pass 4: blank blocks carry no meaning of their own, so they inherit from their neighbour to
|
|
605
|
+
// avoid gaps in a hidden region.
|
|
606
|
+
for (let i = 0; i < decisions.length; i++) {
|
|
607
|
+
const decision = decisions[i];
|
|
608
|
+
const block = blocks[i];
|
|
609
|
+
if (!decision || !block || !block.features.isAllBlank) {
|
|
610
|
+
continue;
|
|
611
|
+
}
|
|
612
|
+
const previous = decisions[i - 1];
|
|
613
|
+
const next = decisions[i + 1];
|
|
614
|
+
if (previous && next && previous.label === next.label) {
|
|
615
|
+
decision.label = previous.label;
|
|
616
|
+
decision.confidence = Math.min(previous.confidence, next.confidence);
|
|
617
|
+
decision.reasons = [...previous.reasons];
|
|
618
|
+
}
|
|
619
|
+
else if (previous) {
|
|
620
|
+
decision.label = previous.label;
|
|
621
|
+
decision.confidence = previous.confidence;
|
|
622
|
+
decision.reasons = [...previous.reasons];
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
// Final guard: a message must never come out as nothing but its own footer.
|
|
626
|
+
//
|
|
627
|
+
// The signature region can legitimately grow to cover everything a sender typed when the
|
|
628
|
+
// message is short. That leaves a reader looking at a collapsed footer and no message, so if no
|
|
629
|
+
// authored block survives, the footer is given back. Being wrong about a signature costs
|
|
630
|
+
// tidiness; being wrong this way costs the whole message.
|
|
631
|
+
const hasVisibleAuthored = decisions.some((decision, index) => {
|
|
632
|
+
const block = blocks[index];
|
|
633
|
+
return block && hasWords(block) && (decision.label === 'NEW_CONTENT' || decision.label === 'INLINE_QUOTE' || decision.label === 'UNKNOWN');
|
|
634
|
+
});
|
|
635
|
+
if (!hasVisibleAuthored) {
|
|
636
|
+
for (let i = 0; i < decisions.length; i++) {
|
|
637
|
+
const decision = decisions[i];
|
|
638
|
+
const block = blocks[i];
|
|
639
|
+
if (decision && block && decision.label === 'SIGNATURE' && hasWords(block)) {
|
|
640
|
+
decision.label = 'NEW_CONTENT';
|
|
641
|
+
decision.confidence = 0.7;
|
|
642
|
+
decision.reasons = ['ambiguous_kept_visible'];
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
return decisions;
|
|
647
|
+
}
|
|
648
|
+
//# sourceMappingURL=rules.js.map
|