@postalsys/email-content 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +207 -17
- package/dist/.tsbuildinfo +1 -1
- package/dist/analyze.d.ts +55 -15
- package/dist/analyze.d.ts.map +1 -1
- package/dist/analyze.js +264 -61
- package/dist/analyze.js.map +1 -1
- package/dist/canonical/html.d.ts +21 -2
- package/dist/canonical/html.d.ts.map +1 -1
- package/dist/canonical/html.js +213 -14
- package/dist/canonical/html.js.map +1 -1
- package/dist/classify/confidence.d.ts +12 -0
- package/dist/classify/confidence.d.ts.map +1 -1
- package/dist/classify/confidence.js +54 -4
- package/dist/classify/confidence.js.map +1 -1
- package/dist/classify/reconcile.d.ts +25 -17
- package/dist/classify/reconcile.d.ts.map +1 -1
- package/dist/classify/reconcile.js +52 -20
- package/dist/classify/reconcile.js.map +1 -1
- package/dist/cli/annotate.d.ts.map +1 -1
- package/dist/cli/annotate.js +6 -3
- package/dist/cli/annotate.js.map +1 -1
- package/dist/cli/args.d.ts +14 -0
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +48 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/benchmark.d.ts +39 -0
- package/dist/cli/benchmark.d.ts.map +1 -1
- package/dist/cli/benchmark.js +179 -66
- package/dist/cli/benchmark.js.map +1 -1
- package/dist/cli/corpus.d.ts.map +1 -1
- package/dist/cli/corpus.js +13 -8
- package/dist/cli/corpus.js.map +1 -1
- package/dist/cli/disagreements.d.ts.map +1 -1
- package/dist/cli/disagreements.js +7 -4
- package/dist/cli/disagreements.js.map +1 -1
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +34 -10
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/extract.d.ts.map +1 -1
- package/dist/cli/extract.js +8 -3
- package/dist/cli/extract.js.map +1 -1
- package/dist/cli/main.js +8 -0
- package/dist/cli/main.js.map +1 -1
- package/dist/corpus/scanner.d.ts +9 -1
- package/dist/corpus/scanner.d.ts.map +1 -1
- package/dist/corpus/scanner.js +52 -13
- package/dist/corpus/scanner.js.map +1 -1
- package/dist/corpus/store.d.ts +33 -2
- package/dist/corpus/store.d.ts.map +1 -1
- package/dist/corpus/store.js +64 -4
- package/dist/corpus/store.js.map +1 -1
- package/dist/detect/patterns.d.ts +29 -0
- package/dist/detect/patterns.d.ts.map +1 -1
- package/dist/detect/patterns.js +80 -6
- package/dist/detect/patterns.js.map +1 -1
- package/dist/detect/reply-header.d.ts +14 -0
- package/dist/detect/reply-header.d.ts.map +1 -1
- package/dist/detect/reply-header.js +80 -37
- package/dist/detect/reply-header.js.map +1 -1
- package/dist/eval/oracle.d.ts +80 -0
- package/dist/eval/oracle.d.ts.map +1 -0
- package/dist/eval/oracle.js +250 -0
- package/dist/eval/oracle.js.map +1 -0
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +94 -56
- package/dist/index.js.map +1 -1
- package/dist/mime/model.d.ts +28 -3
- package/dist/mime/model.d.ts.map +1 -1
- package/dist/mime/model.js +41 -1
- package/dist/mime/model.js.map +1 -1
- package/dist/mime/parse.d.ts.map +1 -1
- package/dist/mime/parse.js +19 -2
- package/dist/mime/parse.js.map +1 -1
- package/dist/mime/select.d.ts +71 -0
- package/dist/mime/select.d.ts.map +1 -0
- package/dist/mime/select.js +198 -0
- package/dist/mime/select.js.map +1 -0
- package/dist/segment/html-blocks.d.ts +1 -1
- package/dist/segment/html-blocks.d.ts.map +1 -1
- package/dist/segment/html-blocks.js +77 -8
- package/dist/segment/html-blocks.js.map +1 -1
- package/dist/segment/text-blocks.d.ts.map +1 -1
- package/dist/segment/text-blocks.js +18 -1
- package/dist/segment/text-blocks.js.map +1 -1
- package/dist/segment/types.d.ts +14 -0
- package/dist/segment/types.d.ts.map +1 -1
- package/dist/types.d.ts +132 -11
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +37 -3
- package/dist/types.js.map +1 -1
- package/dist/util/worker-pool.d.ts +21 -0
- package/dist/util/worker-pool.d.ts.map +1 -1
- package/dist/util/worker-pool.js +61 -13
- package/dist/util/worker-pool.js.map +1 -1
- package/package.json +6 -5
package/dist/corpus/store.d.ts
CHANGED
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
* no native build step and no lockfile that can stop working in two years. The corpus is a few
|
|
6
6
|
* hundred thousand rows of small metadata, which is well within its comfortable range.
|
|
7
7
|
*
|
|
8
|
+
* node:sqlite arrived in Node 22.5, but the extraction library itself runs on Node 20. Loading it
|
|
9
|
+
* lazily keeps that promise: requiring the package never touches this module, so a Node 20 host can
|
|
10
|
+
* use analyze() and extract() while only the corpus tooling needs the newer runtime.
|
|
11
|
+
*
|
|
8
12
|
* The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
|
|
9
13
|
* never copied. See docs/threat-model.md for what the derived data can still reveal.
|
|
10
14
|
*/
|
|
@@ -28,6 +32,15 @@ export interface RunOptions {
|
|
|
28
32
|
root: string;
|
|
29
33
|
[key: string]: unknown;
|
|
30
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Lifecycle of a scan run.
|
|
37
|
+
*
|
|
38
|
+
* `running` also covers a process that died without ever updating its row, which is why release
|
|
39
|
+
* gates must select on `completed` rather than on the newest row. An interrupted run's partial
|
|
40
|
+
* output is not a baseline, and reading it as one silently approves a release against a fraction of
|
|
41
|
+
* the corpus.
|
|
42
|
+
*/
|
|
43
|
+
export type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
|
|
31
44
|
export declare class CorpusStore {
|
|
32
45
|
private readonly db;
|
|
33
46
|
private readonly insertFileStmt;
|
|
@@ -41,20 +54,38 @@ export declare class CorpusStore {
|
|
|
41
54
|
getMeta(key: string): string | null;
|
|
42
55
|
setMeta(key: string, value: string): void;
|
|
43
56
|
startRun(options: RunOptions): number;
|
|
44
|
-
finishRun(runId: number, status:
|
|
57
|
+
finishRun(runId: number, status: RunStatus, counts: {
|
|
45
58
|
seen: number;
|
|
46
59
|
processed: number;
|
|
47
60
|
failed: number;
|
|
48
61
|
skipped: number;
|
|
49
62
|
}): void;
|
|
63
|
+
/**
|
|
64
|
+
* The most recent run that finished cleanly.
|
|
65
|
+
*
|
|
66
|
+
* Every consumer that reads corpus results goes through this rather than taking the newest row.
|
|
67
|
+
* A run that crashed, was interrupted, or is still going has partial output, and treating it as
|
|
68
|
+
* a baseline approves a release against whatever fraction of the corpus happened to finish.
|
|
69
|
+
*/
|
|
70
|
+
latestCompletedRun(): {
|
|
71
|
+
id: number;
|
|
72
|
+
root: string;
|
|
73
|
+
} | null;
|
|
50
74
|
/**
|
|
51
75
|
* True when this exact file has already been processed by the current processing version.
|
|
52
76
|
* Resume relies on size and mtime rather than a hash, because hashing every file would mean
|
|
53
77
|
* reading the whole corpus again just to decide what to skip.
|
|
54
78
|
*/
|
|
55
79
|
isProcessed(relPath: string, size: number, mtime: number): boolean;
|
|
56
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Write a batch inside a single transaction, retrying briefly on lock contention.
|
|
82
|
+
*
|
|
83
|
+
* All or nothing on purpose. A half written batch would leave files recorded as processed whose
|
|
84
|
+
* message rows never landed, and a resumed run would skip them, so the corpus would be quietly
|
|
85
|
+
* short of exactly the messages that were in flight when the write failed.
|
|
86
|
+
*/
|
|
57
87
|
writeBatch(rows: readonly ScanRow[], runId: number): void;
|
|
88
|
+
private writeBatchOnce;
|
|
58
89
|
/** Execute a statement. Returns the driver result, including lastInsertRowid. */
|
|
59
90
|
run(sql: string, ...params: (string | number | null)[]): import("node:sqlite").StatementResultingChanges;
|
|
60
91
|
/** Run a function inside a transaction, rolling back on any throw. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,MAAM,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAuBD;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEvE,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmB;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAE1B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM;IA6DxB,OAAO,CAAC,OAAO;IAkBf,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKnC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIzC,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;IASrC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAM/H;;;;;;OAMG;IACH,kBAAkB,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAIzD;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAQlE;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAqBzD,OAAO,CAAC,cAAc;IA0EtB,iFAAiF;IACjF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;IAItD,sEAAsE;IACtE,WAAW,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI;IAWjC,iFAAiF;IACjF,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE;IAI3F,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI;IAInG,KAAK,IAAI,IAAI;CAQhB"}
|
package/dist/corpus/store.js
CHANGED
|
@@ -6,6 +6,10 @@
|
|
|
6
6
|
* no native build step and no lockfile that can stop working in two years. The corpus is a few
|
|
7
7
|
* hundred thousand rows of small metadata, which is well within its comfortable range.
|
|
8
8
|
*
|
|
9
|
+
* node:sqlite arrived in Node 22.5, but the extraction library itself runs on Node 20. Loading it
|
|
10
|
+
* lazily keeps that promise: requiring the package never touches this module, so a Node 20 host can
|
|
11
|
+
* use analyze() and extract() while only the corpus tooling needs the newer runtime.
|
|
12
|
+
*
|
|
9
13
|
* The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
|
|
10
14
|
* never copied. See docs/threat-model.md for what the derived data can still reveal.
|
|
11
15
|
*/
|
|
@@ -14,12 +18,28 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
14
18
|
};
|
|
15
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
20
|
exports.CorpusStore = void 0;
|
|
17
|
-
const node_sqlite_1 = require("node:sqlite");
|
|
18
21
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
19
22
|
const node_path_1 = __importDefault(require("node:path"));
|
|
20
23
|
const types_1 = require("../types");
|
|
21
24
|
const hash_1 = require("../util/hash");
|
|
22
25
|
const migrations_1 = require("./migrations");
|
|
26
|
+
/**
|
|
27
|
+
* Load node:sqlite, explaining the requirement rather than failing with ERR_UNKNOWN_BUILTIN_MODULE.
|
|
28
|
+
*/
|
|
29
|
+
function loadDatabaseSync() {
|
|
30
|
+
try {
|
|
31
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
32
|
+
return require('node:sqlite').DatabaseSync;
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
throw new Error(`The corpus tooling needs the built in node:sqlite module, added in Node 22.5. This is ${process.version}. ` +
|
|
36
|
+
'The extraction library itself, analyze() and extract(), runs on Node 20 and does not require it.');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** How long SQLite waits for a writer lock before reporting SQLITE_BUSY. */
|
|
40
|
+
const BUSY_TIMEOUT_MS = 10000;
|
|
41
|
+
/** Attempts at a batch write before a lock is treated as a real failure rather than contention. */
|
|
42
|
+
const WRITE_ATTEMPTS = 3;
|
|
23
43
|
class CorpusStore {
|
|
24
44
|
db;
|
|
25
45
|
insertFileStmt;
|
|
@@ -30,10 +50,16 @@ class CorpusStore {
|
|
|
30
50
|
constructor(file) {
|
|
31
51
|
const dir = node_path_1.default.dirname(node_path_1.default.resolve(file));
|
|
32
52
|
node_fs_1.default.mkdirSync(dir, { recursive: true });
|
|
33
|
-
|
|
53
|
+
const DatabaseSync = loadDatabaseSync();
|
|
54
|
+
this.db = new DatabaseSync(file);
|
|
34
55
|
this.db.exec('PRAGMA journal_mode = WAL');
|
|
35
56
|
this.db.exec('PRAGMA synchronous = NORMAL');
|
|
36
57
|
this.db.exec('PRAGMA foreign_keys = ON');
|
|
58
|
+
// WAL allows concurrent readers, but a writer still takes an exclusive lock. Without a busy
|
|
59
|
+
// timeout, a `corpus stats` run that overlaps a scan fails immediately with SQLITE_BUSY and
|
|
60
|
+
// the scan loses the batch it was writing. Ten seconds is far longer than any transaction
|
|
61
|
+
// here, which is bounded by the batch size.
|
|
62
|
+
this.db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
37
63
|
this.migrate();
|
|
38
64
|
this.salt = this.readOrCreateSalt();
|
|
39
65
|
this.insertFileStmt = this.db.prepare(`
|
|
@@ -123,6 +149,16 @@ class CorpusStore {
|
|
|
123
149
|
.prepare('UPDATE runs SET finished_at = ?, status = ?, files_seen = ?, files_processed = ?, files_failed = ?, files_skipped = ? WHERE id = ?')
|
|
124
150
|
.run(Date.now(), status, counts.seen, counts.processed, counts.failed, counts.skipped, runId);
|
|
125
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* The most recent run that finished cleanly.
|
|
154
|
+
*
|
|
155
|
+
* Every consumer that reads corpus results goes through this rather than taking the newest row.
|
|
156
|
+
* A run that crashed, was interrupted, or is still going has partial output, and treating it as
|
|
157
|
+
* a baseline approves a release against whatever fraction of the corpus happened to finish.
|
|
158
|
+
*/
|
|
159
|
+
latestCompletedRun() {
|
|
160
|
+
return this.queryOne("SELECT id, root FROM runs WHERE status = 'completed' ORDER BY id DESC LIMIT 1");
|
|
161
|
+
}
|
|
126
162
|
/**
|
|
127
163
|
* True when this exact file has already been processed by the current processing version.
|
|
128
164
|
* Resume relies on size and mtime rather than a hash, because hashing every file would mean
|
|
@@ -135,12 +171,36 @@ class CorpusStore {
|
|
|
135
171
|
}
|
|
136
172
|
return row.size === size && row.mtime === mtime && row.processing_version === types_1.PROCESSING_VERSION;
|
|
137
173
|
}
|
|
138
|
-
/**
|
|
174
|
+
/**
|
|
175
|
+
* Write a batch inside a single transaction, retrying briefly on lock contention.
|
|
176
|
+
*
|
|
177
|
+
* All or nothing on purpose. A half written batch would leave files recorded as processed whose
|
|
178
|
+
* message rows never landed, and a resumed run would skip them, so the corpus would be quietly
|
|
179
|
+
* short of exactly the messages that were in flight when the write failed.
|
|
180
|
+
*/
|
|
139
181
|
writeBatch(rows, runId) {
|
|
140
182
|
if (!rows.length) {
|
|
141
183
|
return;
|
|
142
184
|
}
|
|
143
|
-
|
|
185
|
+
let attempt = 0;
|
|
186
|
+
for (;;) {
|
|
187
|
+
try {
|
|
188
|
+
this.writeBatchOnce(rows, runId);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
attempt++;
|
|
193
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
194
|
+
// Bounded retry for the one error that is expected and transient. Anything else, and
|
|
195
|
+
// a persistent lock, is a real failure and must reach the caller.
|
|
196
|
+
if (attempt >= WRITE_ATTEMPTS || !/busy|locked/i.test(message)) {
|
|
197
|
+
throw err;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
writeBatchOnce(rows, runId) {
|
|
203
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
144
204
|
try {
|
|
145
205
|
const now = Date.now();
|
|
146
206
|
for (const row of rows) {
|
package/dist/corpus/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":";AAAA
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;;AAEH,sDAAyB;AAEzB,0DAA6B;AAC7B,oCAAiE;AACjE,uCAAuC;AACvC,6CAA0C;AA0B1C;;GAEG;AACH,SAAS,gBAAgB;IACrB,IAAI,CAAC;QACD,iEAAiE;QACjE,OAAQ,OAAO,CAAC,aAAa,CAA+C,CAAC,YAAY,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CACX,yFAAyF,OAAO,CAAC,OAAO,IAAI;YACxG,kGAAkG,CACzG,CAAC;IACN,CAAC;AACL,CAAC;AAED,4EAA4E;AAC5E,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,mGAAmG;AACnG,MAAM,cAAc,GAAG,CAAC,CAAC;AAYzB,MAAa,WAAW;IACH,EAAE,CAAmB;IACrB,cAAc,CAAC;IACf,iBAAiB,CAAC;IAClB,QAAQ,CAAC;IAE1B,8EAA8E;IACrE,IAAI,CAAS;IAEtB,YAAY,IAAY;QACpB,MAAM,GAAG,GAAG,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,iBAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;QACxC,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,4FAA4F;QAC5F,4FAA4F;QAC5F,0FAA0F;QAC1F,4CAA4C;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,eAAe,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEpC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;SAQrC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA4BxC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kEAAkE,CAAC,CAAC;IACxG,CAAC;IAEO,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,CAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAgC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QACvH,KAAK,MAAM,SAAS,IAAI,uBAAU,EAAE,CAAC;YACjC,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC;gBAC/B,SAAS;YACb,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,IAAI,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACL,CAAC;IACL,CAAC;IAEO,gBAAgB;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAmC,CAAC;QAClH,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,KAAK,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,cAAO,GAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,GAAW;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAmC,CAAC;QAC/G,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mGAAmG,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzI,CAAC;IAED,QAAQ,CAAC,OAAmB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,0BAAkB,EAAE,yBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAClH,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC;IAED,SAAS,CAAC,KAAa,EAAE,MAAiB,EAAE,MAA4E;QACpH,IAAI,CAAC,EAAE;aACF,OAAO,CAAC,oIAAoI,CAAC;aAC7I,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACtG,CAAC;IAED;;;;;;OAMG;IACH,kBAAkB;QACd,OAAO,IAAI,CAAC,QAAQ,CAA+B,+EAA+E,CAAC,CAAC;IACxI,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAa;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAA+E,CAAC;QACrH,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,kBAAkB,KAAK,0BAAkB,CAAC;IACrG,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAAwB,EAAE,KAAa;QAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,SAAS,CAAC;YACN,IAAI,CAAC;gBACD,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACjC,OAAO;YACX,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,qFAAqF;gBACrF,kEAAkE;gBAClE,IAAI,OAAO,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7D,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAEO,cAAc,CAAC,IAAwB,EAAE,KAAa;QAC1D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;gBACtB,IAAI,CAAC,cAAc,CAAC,GAAG,CACnB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,UAAU,EACf,GAAG,EACH,KAAK,EACL,0BAAkB,CACrB,CAAC;gBACF,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACtB,IAAI,CAAC,IAAI,EACT,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvB,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnC,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvC,CAAC;YACN,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAW,EAAE,GAAG,MAAkC;QAClD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,sEAAsE;IACtE,WAAW,CAAC,EAAc;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,EAAE,EAAE,CAAC;YACL,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,KAAK,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACjF,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAQ,CAAC;IACtD,CAAC;IAED,QAAQ,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACpF,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAO,IAAI,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK;QACD,IAAI,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACL,oEAAoE;QACxE,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACJ;AA1RD,kCA0RC"}
|
|
@@ -36,6 +36,35 @@ export interface PatternSets {
|
|
|
36
36
|
/** Ids of patterns that failed to compile, for diagnostics. */
|
|
37
37
|
invalid: string[];
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* A classifier asset is missing or corrupt.
|
|
41
|
+
*
|
|
42
|
+
* An installation defect, not malformed email, and therefore deliberately outside the per message
|
|
43
|
+
* fail-soft path. Without the pattern files every detector returns nothing: no reply header is
|
|
44
|
+
* recognised, no quoted history is found, and every message comes back looking like new content.
|
|
45
|
+
* That reads as success, which makes it the worst possible failure mode for a package whose whole
|
|
46
|
+
* job is to know which parts of a message are history.
|
|
47
|
+
*/
|
|
48
|
+
export declare class PatternAssetError extends Error {
|
|
49
|
+
readonly code = "pattern_assets_unusable";
|
|
50
|
+
readonly file: string | null;
|
|
51
|
+
constructor(message: string, file?: string | null);
|
|
52
|
+
}
|
|
53
|
+
/** Total compiled patterns across every set. */
|
|
54
|
+
export declare function countPatterns(sets: PatternSets): number;
|
|
55
|
+
/**
|
|
56
|
+
* Fail loudly when no pattern survived loading.
|
|
57
|
+
*
|
|
58
|
+
* The bundled patterns are read from disk at runtime through path.join(__dirname, 'patterns').
|
|
59
|
+
* @yao-pkg/pkg detects that form and bundles the directory automatically, which is why consumers
|
|
60
|
+
* need no pkg asset configuration, but it is a static analysis heuristic rather than a guarantee.
|
|
61
|
+
* If it ever fails, or the files are missing from a published tarball, every detector quietly
|
|
62
|
+
* returns nothing: no reply header is recognised and reply history stops being detected.
|
|
63
|
+
*
|
|
64
|
+
* The failure direction is at least the safe one, since nothing can be hidden without an anchor.
|
|
65
|
+
* But a build that silently extracts nothing useful should not pass unnoticed.
|
|
66
|
+
*/
|
|
67
|
+
export declare function assertPatternsLoaded(sets: PatternSets, source: string): void;
|
|
39
68
|
/** Load and compile all pattern sets. Cached after the first call. */
|
|
40
69
|
export declare function loadPatterns(overrideDir?: string | null): PatternSets;
|
|
41
70
|
/** Reset the cache. Used by tests that swap pattern directories. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"patterns.d.ts","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,kBAAkB,EAAE,WAAW,EAAE,CAAC;IAClC,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACnC,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAChC,iBAAiB,EAAE,WAAW,EAAE,CAAC;IACjC,aAAa,EAAE,eAAe,EAAE,CAAC;IACjC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;
|
|
1
|
+
{"version":3,"file":"patterns.d.ts","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,kBAAkB,EAAE,WAAW,EAAE,CAAC;IAClC,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACnC,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAChC,iBAAiB,EAAE,WAAW,EAAE,CAAC;IACjC,aAAa,EAAE,eAAe,EAAE,CAAC;IACjC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;AAcD;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IACxC,QAAQ,CAAC,IAAI,6BAA6B;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEjB,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,GAAG,IAAW;CAK1D;AAiGD,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAUvD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAQ5E;AAID,sEAAsE;AACtE,wBAAgB,YAAY,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,WAAW,CA+BrE;AAED,oEAAoE;AACpE,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
|
package/dist/detect/patterns.js
CHANGED
|
@@ -14,29 +14,73 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
14
14
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.PatternAssetError = void 0;
|
|
18
|
+
exports.countPatterns = countPatterns;
|
|
19
|
+
exports.assertPatternsLoaded = assertPatternsLoaded;
|
|
17
20
|
exports.loadPatterns = loadPatterns;
|
|
18
21
|
exports.resetPatternCache = resetPatternCache;
|
|
19
22
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
20
23
|
const node_path_1 = __importDefault(require("node:path"));
|
|
21
24
|
const BUNDLED_DIR = node_path_1.default.join(__dirname, 'patterns');
|
|
25
|
+
/**
|
|
26
|
+
* A classifier asset is missing or corrupt.
|
|
27
|
+
*
|
|
28
|
+
* An installation defect, not malformed email, and therefore deliberately outside the per message
|
|
29
|
+
* fail-soft path. Without the pattern files every detector returns nothing: no reply header is
|
|
30
|
+
* recognised, no quoted history is found, and every message comes back looking like new content.
|
|
31
|
+
* That reads as success, which makes it the worst possible failure mode for a package whose whole
|
|
32
|
+
* job is to know which parts of a message are history.
|
|
33
|
+
*/
|
|
34
|
+
class PatternAssetError extends Error {
|
|
35
|
+
code = 'pattern_assets_unusable';
|
|
36
|
+
file;
|
|
37
|
+
constructor(message, file = null) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = 'PatternAssetError';
|
|
40
|
+
this.file = file;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.PatternAssetError = PatternAssetError;
|
|
22
44
|
function readJson(dir, file) {
|
|
45
|
+
let raw;
|
|
46
|
+
try {
|
|
47
|
+
raw = node_fs_1.default.readFileSync(node_path_1.default.join(dir, file), 'utf8');
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return { ok: false, reason: 'missing' };
|
|
51
|
+
}
|
|
52
|
+
// Present but unreadable is a different fault from absent, and must not be silently treated as
|
|
53
|
+
// an empty pattern set. A truncated file in a published tarball produced exactly that.
|
|
23
54
|
try {
|
|
24
|
-
const raw = node_fs_1.default.readFileSync(node_path_1.default.join(dir, file), 'utf8');
|
|
25
55
|
const parsed = JSON.parse(raw);
|
|
26
|
-
|
|
56
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
57
|
+
return { ok: false, reason: 'invalid' };
|
|
58
|
+
}
|
|
59
|
+
return { ok: true, value: parsed };
|
|
27
60
|
}
|
|
28
61
|
catch {
|
|
29
|
-
return
|
|
62
|
+
return { ok: false, reason: 'invalid' };
|
|
30
63
|
}
|
|
31
64
|
}
|
|
32
65
|
function loadFile(file, overrideDir) {
|
|
33
66
|
if (overrideDir) {
|
|
34
67
|
const override = readJson(overrideDir, file);
|
|
35
|
-
if (override) {
|
|
36
|
-
return override;
|
|
68
|
+
if (override.ok) {
|
|
69
|
+
return override.value;
|
|
37
70
|
}
|
|
71
|
+
if (override.reason === 'invalid') {
|
|
72
|
+
throw new PatternAssetError(`Pattern file ${file} in ${overrideDir} is not valid JSON.`, file);
|
|
73
|
+
}
|
|
74
|
+
// Absent from an override directory is normal: an override may replace only some files.
|
|
75
|
+
}
|
|
76
|
+
const bundled = readJson(BUNDLED_DIR, file);
|
|
77
|
+
if (bundled.ok) {
|
|
78
|
+
return bundled.value;
|
|
79
|
+
}
|
|
80
|
+
if (bundled.reason === 'invalid') {
|
|
81
|
+
throw new PatternAssetError(`Bundled pattern file ${file} is corrupt. Reinstall the package.`, file);
|
|
38
82
|
}
|
|
39
|
-
return
|
|
83
|
+
return {};
|
|
40
84
|
}
|
|
41
85
|
function compileLinePatterns(input, invalid) {
|
|
42
86
|
if (!Array.isArray(input)) {
|
|
@@ -89,6 +133,35 @@ function compileSelectors(input, invalid) {
|
|
|
89
133
|
}
|
|
90
134
|
return out;
|
|
91
135
|
}
|
|
136
|
+
/** Total compiled patterns across every set. */
|
|
137
|
+
function countPatterns(sets) {
|
|
138
|
+
return (sets.replyHeaders.length +
|
|
139
|
+
sets.forwardHeaders.length +
|
|
140
|
+
sets.quotedHeaderFields.length +
|
|
141
|
+
sets.signatureDelimiters.length +
|
|
142
|
+
sets.signaturePhrases.length +
|
|
143
|
+
sets.disclaimerPhrases.length +
|
|
144
|
+
sets.htmlSelectors.length);
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Fail loudly when no pattern survived loading.
|
|
148
|
+
*
|
|
149
|
+
* The bundled patterns are read from disk at runtime through path.join(__dirname, 'patterns').
|
|
150
|
+
* @yao-pkg/pkg detects that form and bundles the directory automatically, which is why consumers
|
|
151
|
+
* need no pkg asset configuration, but it is a static analysis heuristic rather than a guarantee.
|
|
152
|
+
* If it ever fails, or the files are missing from a published tarball, every detector quietly
|
|
153
|
+
* returns nothing: no reply header is recognised and reply history stops being detected.
|
|
154
|
+
*
|
|
155
|
+
* The failure direction is at least the safe one, since nothing can be hidden without an anchor.
|
|
156
|
+
* But a build that silently extracts nothing useful should not pass unnoticed.
|
|
157
|
+
*/
|
|
158
|
+
function assertPatternsLoaded(sets, source) {
|
|
159
|
+
if (countPatterns(sets) === 0) {
|
|
160
|
+
throw new PatternAssetError(`No detector patterns could be loaded from ${source}. ` +
|
|
161
|
+
'The pattern JSON files are missing from the installation or the bundle. ' +
|
|
162
|
+
'Without them no reply history can be detected.');
|
|
163
|
+
}
|
|
164
|
+
}
|
|
92
165
|
let cached = null;
|
|
93
166
|
/** Load and compile all pattern sets. Cached after the first call. */
|
|
94
167
|
function loadPatterns(overrideDir) {
|
|
@@ -113,6 +186,7 @@ function loadPatterns(overrideDir) {
|
|
|
113
186
|
htmlSelectors: compileSelectors(html.selectors, invalid),
|
|
114
187
|
invalid
|
|
115
188
|
};
|
|
189
|
+
assertPatternsLoaded(sets, dir ?? BUNDLED_DIR);
|
|
116
190
|
if (overrideDir === undefined) {
|
|
117
191
|
cached = sets;
|
|
118
192
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG
|
|
1
|
+
{"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;AAmKH,sCAUC;AAcD,oDAQC;AAKD,oCA+BC;AAGD,8CAEC;AA1OD,sDAAyB;AACzB,0DAA6B;AA0C7B,MAAM,WAAW,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAa,iBAAkB,SAAQ,KAAK;IAC/B,IAAI,GAAG,yBAAyB,CAAC;IACjC,IAAI,CAAgB;IAE7B,YAAY,OAAe,EAAE,OAAsB,IAAI;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AATD,8CASC;AAID,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IACD,+FAA+F;IAC/F,uFAAuF;IACvF,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAiC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,WAA0B;IACtD,IAAI,WAAW,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC,KAAK,CAAC;QAC1B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,iBAAiB,CAAC,gBAAgB,IAAI,OAAO,WAAW,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACnG,CAAC;QACD,wFAAwF;IAC5F,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,OAAO,CAAC,KAAK,CAAC;IACzB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,iBAAiB,CAAC,wBAAwB,IAAI,qCAAqC,EAAE,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,OAAiB;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAqB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,GAAG,CAAC,IAAI,CAAC;gBACL,EAAE;gBACF,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACjE,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBACjE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9E,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,OAAiB;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAkC,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC;YACL,EAAE;YACF,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;YAC3D,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAA4B;YACrF,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YACjE,QAAQ;YACR,UAAU,EAAE,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAgB,aAAa,CAAC,IAAiB;IAC3C,OAAO,CACH,IAAI,CAAC,YAAY,CAAC,MAAM;QACxB,IAAI,CAAC,cAAc,CAAC,MAAM;QAC1B,IAAI,CAAC,kBAAkB,CAAC,MAAM;QAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM;QAC/B,IAAI,CAAC,gBAAgB,CAAC,MAAM;QAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5B,CAAC;AACN,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,oBAAoB,CAAC,IAAiB,EAAE,MAAc;IAClE,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,iBAAiB,CACvB,6CAA6C,MAAM,IAAI;YACnD,0EAA0E;YAC1E,gDAAgD,CACvD,CAAC;IACN,CAAC;AACL,CAAC;AAED,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,sEAAsE;AACtE,SAAgB,YAAY,CAAC,WAA2B;IACpD,IAAI,MAAM,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;IACjG,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,QAAQ,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAgB;QACtB,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC1D,cAAc,EAAE,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC9D,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QAC/D,mBAAmB,EAAE,mBAAmB,CAAC,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC;QACxE,gBAAgB,EAAE,mBAAmB,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;QAClE,iBAAiB,EAAE,mBAAmB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;QACpE,aAAa,EAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;QACxD,OAAO;KACV,CAAC;IAEF,oBAAoB,CAAC,IAAI,EAAE,GAAG,IAAI,WAAW,CAAC,CAAC;IAE/C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,oEAAoE;AACpE,SAAgB,iBAAiB;IAC7B,MAAM,GAAG,IAAI,CAAC;AAClB,CAAC"}
|
|
@@ -32,6 +32,20 @@ export declare function detectAttributionLine(line: string): Evidence | null;
|
|
|
32
32
|
*
|
|
33
33
|
* Returns the first match found, scanning forward. The caller supplies de-prefixed line content so
|
|
34
34
|
* that a header block inside a quote is detected the same way as one outside it.
|
|
35
|
+
*
|
|
36
|
+
* Single pass, deliberately. The obvious implementation restarts an inner forward scan at every
|
|
37
|
+
* candidate line, which is quadratic: 12,000 consecutive `Cc:` lines never form a strong block but
|
|
38
|
+
* make the inner scan traverse almost the whole remaining input from every outer position, and
|
|
39
|
+
* measured at 4.8 seconds of blocked event loop on that input. Inbound mail is attacker controlled,
|
|
40
|
+
* so that is a denial of service, not a slow path.
|
|
41
|
+
*
|
|
42
|
+
* The single pass is exactly equivalent, not an approximation. Lines partition into *chains*:
|
|
43
|
+
* maximal runs of header-shaped lines joined by blank gaps of at most `MAX_GAP`, terminated by any
|
|
44
|
+
* other line or by a longer blank run. The set of roles collected from a start inside a chain is a
|
|
45
|
+
* subset of the set collected from the chain's own start, and strength is monotone under superset,
|
|
46
|
+
* so if the chain start is not strong no later start in that chain can be, and if it is strong it
|
|
47
|
+
* is also the earliest strong start. Checking each chain once therefore returns the same match the
|
|
48
|
+
* restarting scan did.
|
|
35
49
|
*/
|
|
36
50
|
export declare function detectQuotedHeaderBlock(lines: readonly string[], from?: number): ReplyHeaderMatch | null;
|
|
37
51
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AA4BjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAgBnE;
|
|
1
|
+
{"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AA4BjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAgBnE;AASD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,SAAI,GAAG,gBAAgB,GAAG,IAAI,CA8EnG;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CASjE"}
|
|
@@ -61,57 +61,100 @@ function detectAttributionLine(line) {
|
|
|
61
61
|
}
|
|
62
62
|
return best;
|
|
63
63
|
}
|
|
64
|
+
/** Is this run of header-shaped lines convincing enough to anchor a region? */
|
|
65
|
+
function isStrong(roles) {
|
|
66
|
+
// A convincing block names the originator plus at least a date or a subject, so that a
|
|
67
|
+
// pasted "To: ... Cc: ..." address list on its own does not qualify.
|
|
68
|
+
return roles.size >= MIN_FIELDS && roles.has('originator') && (roles.has('date') || roles.has('subject'));
|
|
69
|
+
}
|
|
64
70
|
/**
|
|
65
71
|
* Detect an embedded header block starting at or after `from`.
|
|
66
72
|
*
|
|
67
73
|
* Returns the first match found, scanning forward. The caller supplies de-prefixed line content so
|
|
68
74
|
* that a header block inside a quote is detected the same way as one outside it.
|
|
75
|
+
*
|
|
76
|
+
* Single pass, deliberately. The obvious implementation restarts an inner forward scan at every
|
|
77
|
+
* candidate line, which is quadratic: 12,000 consecutive `Cc:` lines never form a strong block but
|
|
78
|
+
* make the inner scan traverse almost the whole remaining input from every outer position, and
|
|
79
|
+
* measured at 4.8 seconds of blocked event loop on that input. Inbound mail is attacker controlled,
|
|
80
|
+
* so that is a denial of service, not a slow path.
|
|
81
|
+
*
|
|
82
|
+
* The single pass is exactly equivalent, not an approximation. Lines partition into *chains*:
|
|
83
|
+
* maximal runs of header-shaped lines joined by blank gaps of at most `MAX_GAP`, terminated by any
|
|
84
|
+
* other line or by a longer blank run. The set of roles collected from a start inside a chain is a
|
|
85
|
+
* subset of the set collected from the chain's own start, and strength is monotone under superset,
|
|
86
|
+
* so if the chain start is not strong no later start in that chain can be, and if it is strong it
|
|
87
|
+
* is also the earliest strong start. Checking each chain once therefore returns the same match the
|
|
88
|
+
* restarting scan did.
|
|
69
89
|
*/
|
|
70
90
|
function detectQuotedHeaderBlock(lines, from = 0) {
|
|
71
91
|
const patterns = (0, patterns_1.loadPatterns)();
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
92
|
+
const evidence = {
|
|
93
|
+
id: 'reply:quoted_header_block',
|
|
94
|
+
kind: 'reply-anchor',
|
|
95
|
+
// Below a single line attribution on purpose: a person can paste this shape.
|
|
96
|
+
strength: 0.88
|
|
97
|
+
};
|
|
98
|
+
// Start of the chain currently being accumulated, which may be a blank line: the restarting
|
|
99
|
+
// scan reported the outer index it began at, and that index is allowed to sit on the at most
|
|
100
|
+
// MAX_GAP blank lines that lead into the first field.
|
|
101
|
+
let chainStart = -1;
|
|
102
|
+
let roles = new Set();
|
|
103
|
+
let last = -1;
|
|
104
|
+
let gap = 0;
|
|
105
|
+
for (let j = Math.max(0, from); j < lines.length; j++) {
|
|
106
|
+
const line = (lines[j] ?? '').trim();
|
|
107
|
+
if (!line) {
|
|
108
|
+
if (chainStart === -1) {
|
|
109
|
+
chainStart = j;
|
|
110
|
+
gap = 1;
|
|
83
111
|
continue;
|
|
84
112
|
}
|
|
85
|
-
|
|
86
|
-
if (
|
|
87
|
-
|
|
113
|
+
gap++;
|
|
114
|
+
if (gap <= MAX_GAP) {
|
|
115
|
+
continue;
|
|
88
116
|
}
|
|
89
|
-
gap
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
//
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
117
|
+
// The blank run is now too long to be an internal gap. Close the chain, then begin a
|
|
118
|
+
// new tentative one far enough into the blank run that exactly MAX_GAP blanks can still
|
|
119
|
+
// precede a following field line, which is where the restarting scan would have found
|
|
120
|
+
// its next viable start.
|
|
121
|
+
if (isStrong(roles) && last >= chainStart) {
|
|
122
|
+
return { startLine: chainStart, endLine: last, evidence };
|
|
123
|
+
}
|
|
124
|
+
roles = new Set();
|
|
125
|
+
last = -1;
|
|
126
|
+
chainStart = j - MAX_GAP + 1;
|
|
127
|
+
gap = MAX_GAP;
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const matched = patterns.quotedHeaderFields.find(pattern => pattern.regex.test(line));
|
|
131
|
+
if (!matched) {
|
|
132
|
+
// Ordinary content terminates the chain outright.
|
|
133
|
+
if (isStrong(roles) && last >= chainStart && chainStart !== -1) {
|
|
134
|
+
return { startLine: chainStart, endLine: last, evidence };
|
|
97
135
|
}
|
|
98
|
-
|
|
136
|
+
roles = new Set();
|
|
137
|
+
last = -1;
|
|
138
|
+
chainStart = -1;
|
|
139
|
+
gap = 0;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (chainStart === -1) {
|
|
143
|
+
chainStart = j;
|
|
99
144
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
id: 'reply:quoted_header_block',
|
|
109
|
-
kind: 'reply-anchor',
|
|
110
|
-
// Below a single line attribution on purpose: a person can paste this shape.
|
|
111
|
-
strength: 0.88
|
|
112
|
-
}
|
|
113
|
-
};
|
|
145
|
+
gap = 0;
|
|
146
|
+
const role = matched.role || matched.id;
|
|
147
|
+
const value = valueOf(line);
|
|
148
|
+
// Only fields whose value looks like a real header value count towards the threshold.
|
|
149
|
+
// The line still continues the block either way, so a "Subject:" with unusual content
|
|
150
|
+
// does not truncate an otherwise convincing header block.
|
|
151
|
+
if (role === 'originator' ? ORIGINATOR_VALUE.test(value) : role === 'date' ? DATE_VALUE.test(value) : true) {
|
|
152
|
+
roles.add(role);
|
|
114
153
|
}
|
|
154
|
+
last = j;
|
|
155
|
+
}
|
|
156
|
+
if (chainStart !== -1 && isStrong(roles) && last >= chainStart) {
|
|
157
|
+
return { startLine: chainStart, endLine: last, evidence };
|
|
115
158
|
}
|
|
116
159
|
return null;
|
|
117
160
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AA6CH,sDAgBC;
|
|
1
|
+
{"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AA6CH,sDAgBC;AA6BD,0DA8EC;AAQD,kDASC;AAvLD,yCAA0C;AAG1C,4EAA4E;AAC5E,MAAM,OAAO,GAAG,CAAC,CAAC;AAElB,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;GAQG;AACH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF,iGAAiG;AACjG,MAAM,UAAU,GACZ,sUAAsU,CAAC;AAE3U,SAAS,OAAO,CAAC,IAAY;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5D,CAAC;AAUD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5C,IAAI,GAAG,EAAE,EAAE,EAAE,SAAS,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,SAAS,QAAQ,CAAC,KAA0B;IACxC,uFAAuF;IACvF,qEAAqE;IACrE,OAAO,KAAK,CAAC,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,uBAAuB,CAAC,KAAwB,EAAE,IAAI,GAAG,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAEhC,MAAM,QAAQ,GAAa;QACvB,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,cAAc;QACpB,6EAA6E;QAC7E,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,sDAAsD;IACtD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAErC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,UAAU,GAAG,CAAC,CAAC;gBACf,GAAG,GAAG,CAAC,CAAC;gBACR,SAAS;YACb,CAAC;YACD,GAAG,EAAE,CAAC;YACN,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,qFAAqF;YACrF,wFAAwF;YACxF,sFAAsF;YACtF,yBAAyB;YACzB,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;gBACxC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;YAC7B,GAAG,GAAG,OAAO,CAAC;YACd,SAAS;QACb,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,CAAC,CAAC;YAChB,GAAG,GAAG,CAAC,CAAC;YACR,SAAS;QACb,CAAC;QAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,UAAU,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,GAAG,GAAG,CAAC,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,sFAAsF;QACtF,sFAAsF;QACtF,0DAA0D;QAC1D,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;QAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9D,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACrF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}
|