@postalsys/email-content 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +207 -17
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/analyze.d.ts +55 -15
  4. package/dist/analyze.d.ts.map +1 -1
  5. package/dist/analyze.js +264 -61
  6. package/dist/analyze.js.map +1 -1
  7. package/dist/canonical/html.d.ts +21 -2
  8. package/dist/canonical/html.d.ts.map +1 -1
  9. package/dist/canonical/html.js +213 -14
  10. package/dist/canonical/html.js.map +1 -1
  11. package/dist/classify/confidence.d.ts +12 -0
  12. package/dist/classify/confidence.d.ts.map +1 -1
  13. package/dist/classify/confidence.js +54 -4
  14. package/dist/classify/confidence.js.map +1 -1
  15. package/dist/classify/reconcile.d.ts +25 -17
  16. package/dist/classify/reconcile.d.ts.map +1 -1
  17. package/dist/classify/reconcile.js +52 -20
  18. package/dist/classify/reconcile.js.map +1 -1
  19. package/dist/cli/annotate.d.ts.map +1 -1
  20. package/dist/cli/annotate.js +6 -3
  21. package/dist/cli/annotate.js.map +1 -1
  22. package/dist/cli/args.d.ts +14 -0
  23. package/dist/cli/args.d.ts.map +1 -1
  24. package/dist/cli/args.js +48 -1
  25. package/dist/cli/args.js.map +1 -1
  26. package/dist/cli/benchmark.d.ts +39 -0
  27. package/dist/cli/benchmark.d.ts.map +1 -1
  28. package/dist/cli/benchmark.js +179 -66
  29. package/dist/cli/benchmark.js.map +1 -1
  30. package/dist/cli/corpus.d.ts.map +1 -1
  31. package/dist/cli/corpus.js +13 -8
  32. package/dist/cli/corpus.js.map +1 -1
  33. package/dist/cli/disagreements.d.ts.map +1 -1
  34. package/dist/cli/disagreements.js +7 -4
  35. package/dist/cli/disagreements.js.map +1 -1
  36. package/dist/cli/evaluate.d.ts.map +1 -1
  37. package/dist/cli/evaluate.js +34 -10
  38. package/dist/cli/evaluate.js.map +1 -1
  39. package/dist/cli/extract.d.ts.map +1 -1
  40. package/dist/cli/extract.js +8 -3
  41. package/dist/cli/extract.js.map +1 -1
  42. package/dist/cli/main.js +8 -0
  43. package/dist/cli/main.js.map +1 -1
  44. package/dist/corpus/scanner.d.ts +9 -1
  45. package/dist/corpus/scanner.d.ts.map +1 -1
  46. package/dist/corpus/scanner.js +52 -13
  47. package/dist/corpus/scanner.js.map +1 -1
  48. package/dist/corpus/store.d.ts +33 -2
  49. package/dist/corpus/store.d.ts.map +1 -1
  50. package/dist/corpus/store.js +64 -4
  51. package/dist/corpus/store.js.map +1 -1
  52. package/dist/detect/patterns.d.ts +29 -0
  53. package/dist/detect/patterns.d.ts.map +1 -1
  54. package/dist/detect/patterns.js +80 -6
  55. package/dist/detect/patterns.js.map +1 -1
  56. package/dist/detect/reply-header.d.ts +14 -0
  57. package/dist/detect/reply-header.d.ts.map +1 -1
  58. package/dist/detect/reply-header.js +80 -37
  59. package/dist/detect/reply-header.js.map +1 -1
  60. package/dist/eval/oracle.d.ts +80 -0
  61. package/dist/eval/oracle.d.ts.map +1 -0
  62. package/dist/eval/oracle.js +250 -0
  63. package/dist/eval/oracle.js.map +1 -0
  64. package/dist/index.d.ts +6 -3
  65. package/dist/index.d.ts.map +1 -1
  66. package/dist/index.js +94 -56
  67. package/dist/index.js.map +1 -1
  68. package/dist/mime/model.d.ts +28 -3
  69. package/dist/mime/model.d.ts.map +1 -1
  70. package/dist/mime/model.js +41 -1
  71. package/dist/mime/model.js.map +1 -1
  72. package/dist/mime/parse.d.ts.map +1 -1
  73. package/dist/mime/parse.js +19 -2
  74. package/dist/mime/parse.js.map +1 -1
  75. package/dist/mime/select.d.ts +71 -0
  76. package/dist/mime/select.d.ts.map +1 -0
  77. package/dist/mime/select.js +198 -0
  78. package/dist/mime/select.js.map +1 -0
  79. package/dist/segment/html-blocks.d.ts +1 -1
  80. package/dist/segment/html-blocks.d.ts.map +1 -1
  81. package/dist/segment/html-blocks.js +77 -8
  82. package/dist/segment/html-blocks.js.map +1 -1
  83. package/dist/segment/text-blocks.d.ts.map +1 -1
  84. package/dist/segment/text-blocks.js +18 -1
  85. package/dist/segment/text-blocks.js.map +1 -1
  86. package/dist/segment/types.d.ts +14 -0
  87. package/dist/segment/types.d.ts.map +1 -1
  88. package/dist/types.d.ts +132 -11
  89. package/dist/types.d.ts.map +1 -1
  90. package/dist/types.js +37 -3
  91. package/dist/types.js.map +1 -1
  92. package/dist/util/worker-pool.d.ts +21 -0
  93. package/dist/util/worker-pool.d.ts.map +1 -1
  94. package/dist/util/worker-pool.js +61 -13
  95. package/dist/util/worker-pool.js.map +1 -1
  96. package/package.json +6 -5
@@ -5,6 +5,10 @@
5
5
  * no native build step and no lockfile that can stop working in two years. The corpus is a few
6
6
  * hundred thousand rows of small metadata, which is well within its comfortable range.
7
7
  *
8
+ * node:sqlite arrived in Node 22.5, but the extraction library itself runs on Node 20. Loading it
9
+ * lazily keeps that promise: requiring the package never touches this module, so a Node 20 host can
10
+ * use analyze() and extract() while only the corpus tooling needs the newer runtime.
11
+ *
8
12
  * The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
9
13
  * never copied. See docs/threat-model.md for what the derived data can still reveal.
10
14
  */
@@ -28,6 +32,15 @@ export interface RunOptions {
28
32
  root: string;
29
33
  [key: string]: unknown;
30
34
  }
35
+ /**
36
+ * Lifecycle of a scan run.
37
+ *
38
+ * `running` also covers a process that died without ever updating its row, which is why release
39
+ * gates must select on `completed` rather than on the newest row. An interrupted run's partial
40
+ * output is not a baseline, and reading it as one silently approves a release against a fraction of
41
+ * the corpus.
42
+ */
43
+ export type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
31
44
  export declare class CorpusStore {
32
45
  private readonly db;
33
46
  private readonly insertFileStmt;
@@ -41,20 +54,38 @@ export declare class CorpusStore {
41
54
  getMeta(key: string): string | null;
42
55
  setMeta(key: string, value: string): void;
43
56
  startRun(options: RunOptions): number;
44
- finishRun(runId: number, status: 'completed' | 'interrupted' | 'failed', counts: {
57
+ finishRun(runId: number, status: RunStatus, counts: {
45
58
  seen: number;
46
59
  processed: number;
47
60
  failed: number;
48
61
  skipped: number;
49
62
  }): void;
63
+ /**
64
+ * The most recent run that finished cleanly.
65
+ *
66
+ * Every consumer that reads corpus results goes through this rather than taking the newest row.
67
+ * A run that crashed, was interrupted, or is still going has partial output, and treating it as
68
+ * a baseline approves a release against whatever fraction of the corpus happened to finish.
69
+ */
70
+ latestCompletedRun(): {
71
+ id: number;
72
+ root: string;
73
+ } | null;
50
74
  /**
51
75
  * True when this exact file has already been processed by the current processing version.
52
76
  * Resume relies on size and mtime rather than a hash, because hashing every file would mean
53
77
  * reading the whole corpus again just to decide what to skip.
54
78
  */
55
79
  isProcessed(relPath: string, size: number, mtime: number): boolean;
56
- /** Write a batch inside a single transaction. */
80
+ /**
81
+ * Write a batch inside a single transaction, retrying briefly on lock contention.
82
+ *
83
+ * All or nothing on purpose. A half written batch would leave files recorded as processed whose
84
+ * message rows never landed, and a resumed run would skip them, so the corpus would be quietly
85
+ * short of exactly the messages that were in flight when the write failed.
86
+ */
57
87
  writeBatch(rows: readonly ScanRow[], runId: number): void;
88
+ private writeBatchOnce;
58
89
  /** Execute a statement. Returns the driver result, including lastInsertRowid. */
59
90
  run(sql: string, ...params: (string | number | null)[]): import("node:sqlite").StatementResultingChanges;
60
91
  /** Run a function inside a transaction, rolling back on any throw. */
@@ -1 +1 @@
1
- {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,MAAM,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAe;IAClC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAE1B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM;IAuDxB,OAAO,CAAC,OAAO;IAkBf,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKnC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIzC,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;IASrC,SAAS,CACL,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,QAAQ,EAC9C,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAC7E,IAAI;IAMP;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAQlE,iDAAiD;IACjD,UAAU,CAAC,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IA6EzD,iFAAiF;IACjF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;IAItD,sEAAsE;IACtE,WAAW,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI;IAWjC,iFAAiF;IACjF,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE;IAI3F,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI;IAInG,KAAK,IAAI,IAAI;CAQhB"}
1
+ {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,MAAM,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,OAAO;IACpB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAuBD;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEvE,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmB;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAE1B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM;IA6DxB,OAAO,CAAC,OAAO;IAkBf,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKnC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAIzC,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;IASrC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAM/H;;;;;;OAMG;IACH,kBAAkB,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAIzD;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAQlE;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAqBzD,OAAO,CAAC,cAAc;IA0EtB,iFAAiF;IACjF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;IAItD,sEAAsE;IACtE,WAAW,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI;IAWjC,iFAAiF;IACjF,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE;IAI3F,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI;IAInG,KAAK,IAAI,IAAI;CAQhB"}
@@ -6,6 +6,10 @@
6
6
  * no native build step and no lockfile that can stop working in two years. The corpus is a few
7
7
  * hundred thousand rows of small metadata, which is well within its comfortable range.
8
8
  *
9
+ * node:sqlite arrived in Node 22.5, but the extraction library itself runs on Node 20. Loading it
10
+ * lazily keeps that promise: requiring the package never touches this module, so a Node 20 host can
11
+ * use analyze() and extract() while only the corpus tooling needs the newer runtime.
12
+ *
9
13
  * The store holds derived metadata only. Raw messages stay on disk, referenced by path, and are
10
14
  * never copied. See docs/threat-model.md for what the derived data can still reveal.
11
15
  */
@@ -14,12 +18,28 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
14
18
  };
15
19
  Object.defineProperty(exports, "__esModule", { value: true });
16
20
  exports.CorpusStore = void 0;
17
- const node_sqlite_1 = require("node:sqlite");
18
21
  const node_fs_1 = __importDefault(require("node:fs"));
19
22
  const node_path_1 = __importDefault(require("node:path"));
20
23
  const types_1 = require("../types");
21
24
  const hash_1 = require("../util/hash");
22
25
  const migrations_1 = require("./migrations");
26
+ /**
27
+ * Load node:sqlite, explaining the requirement rather than failing with ERR_UNKNOWN_BUILTIN_MODULE.
28
+ */
29
+ function loadDatabaseSync() {
30
+ try {
31
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
32
+ return require('node:sqlite').DatabaseSync;
33
+ }
34
+ catch {
35
+ throw new Error(`The corpus tooling needs the built in node:sqlite module, added in Node 22.5. This is ${process.version}. ` +
36
+ 'The extraction library itself, analyze() and extract(), runs on Node 20 and does not require it.');
37
+ }
38
+ }
39
+ /** How long SQLite waits for a writer lock before reporting SQLITE_BUSY. */
40
+ const BUSY_TIMEOUT_MS = 10000;
41
+ /** Attempts at a batch write before a lock is treated as a real failure rather than contention. */
42
+ const WRITE_ATTEMPTS = 3;
23
43
  class CorpusStore {
24
44
  db;
25
45
  insertFileStmt;
@@ -30,10 +50,16 @@ class CorpusStore {
30
50
  constructor(file) {
31
51
  const dir = node_path_1.default.dirname(node_path_1.default.resolve(file));
32
52
  node_fs_1.default.mkdirSync(dir, { recursive: true });
33
- this.db = new node_sqlite_1.DatabaseSync(file);
53
+ const DatabaseSync = loadDatabaseSync();
54
+ this.db = new DatabaseSync(file);
34
55
  this.db.exec('PRAGMA journal_mode = WAL');
35
56
  this.db.exec('PRAGMA synchronous = NORMAL');
36
57
  this.db.exec('PRAGMA foreign_keys = ON');
58
+ // WAL allows concurrent readers, but a writer still takes an exclusive lock. Without a busy
59
+ // timeout, a `corpus stats` run that overlaps a scan fails immediately with SQLITE_BUSY and
60
+ // the scan loses the batch it was writing. Ten seconds is far longer than any transaction
61
+ // here, which is bounded by the batch size.
62
+ this.db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
37
63
  this.migrate();
38
64
  this.salt = this.readOrCreateSalt();
39
65
  this.insertFileStmt = this.db.prepare(`
@@ -123,6 +149,16 @@ class CorpusStore {
123
149
  .prepare('UPDATE runs SET finished_at = ?, status = ?, files_seen = ?, files_processed = ?, files_failed = ?, files_skipped = ? WHERE id = ?')
124
150
  .run(Date.now(), status, counts.seen, counts.processed, counts.failed, counts.skipped, runId);
125
151
  }
152
+ /**
153
+ * The most recent run that finished cleanly.
154
+ *
155
+ * Every consumer that reads corpus results goes through this rather than taking the newest row.
156
+ * A run that crashed, was interrupted, or is still going has partial output, and treating it as
157
+ * a baseline approves a release against whatever fraction of the corpus happened to finish.
158
+ */
159
+ latestCompletedRun() {
160
+ return this.queryOne("SELECT id, root FROM runs WHERE status = 'completed' ORDER BY id DESC LIMIT 1");
161
+ }
126
162
  /**
127
163
  * True when this exact file has already been processed by the current processing version.
128
164
  * Resume relies on size and mtime rather than a hash, because hashing every file would mean
@@ -135,12 +171,36 @@ class CorpusStore {
135
171
  }
136
172
  return row.size === size && row.mtime === mtime && row.processing_version === types_1.PROCESSING_VERSION;
137
173
  }
138
- /** Write a batch inside a single transaction. */
174
+ /**
175
+ * Write a batch inside a single transaction, retrying briefly on lock contention.
176
+ *
177
+ * All or nothing on purpose. A half written batch would leave files recorded as processed whose
178
+ * message rows never landed, and a resumed run would skip them, so the corpus would be quietly
179
+ * short of exactly the messages that were in flight when the write failed.
180
+ */
139
181
  writeBatch(rows, runId) {
140
182
  if (!rows.length) {
141
183
  return;
142
184
  }
143
- this.db.exec('BEGIN');
185
+ let attempt = 0;
186
+ for (;;) {
187
+ try {
188
+ this.writeBatchOnce(rows, runId);
189
+ return;
190
+ }
191
+ catch (err) {
192
+ attempt++;
193
+ const message = err instanceof Error ? err.message : String(err);
194
+ // Bounded retry for the one error that is expected and transient. Anything else, and
195
+ // a persistent lock, is a real failure and must reach the caller.
196
+ if (attempt >= WRITE_ATTEMPTS || !/busy|locked/i.test(message)) {
197
+ throw err;
198
+ }
199
+ }
200
+ }
201
+ }
202
+ writeBatchOnce(rows, runId) {
203
+ this.db.exec('BEGIN IMMEDIATE');
144
204
  try {
145
205
  const now = Date.now();
146
206
  for (const row of rows) {
@@ -1 +1 @@
1
- {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;;;;AAEH,6CAA2C;AAC3C,sDAAyB;AACzB,0DAA6B;AAC7B,oCAAiE;AACjE,uCAAuC;AACvC,6CAA0C;AA0B1C,MAAa,WAAW;IACH,EAAE,CAAe;IACjB,cAAc,CAAC;IACf,iBAAiB,CAAC;IAClB,QAAQ,CAAC;IAE1B,8EAA8E;IACrE,IAAI,CAAS;IAEtB,YAAY,IAAY;QACpB,MAAM,GAAG,GAAG,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,iBAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,IAAI,CAAC,EAAE,GAAG,IAAI,0BAAY,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEpC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;SAQrC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA4BxC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kEAAkE,CAAC,CAAC;IACxG,CAAC;IAEO,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,CAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAgC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QACvH,KAAK,MAAM,SAAS,IAAI,uBAAU,EAAE,CAAC;YACjC,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC;gBAC/B,SAAS;YACb,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,IAAI,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACL,CAAC;IACL,CAAC;IAEO,gBAAgB;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAmC,CAAC;QAClH,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,KAAK,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,cAAO,GAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,GAAW;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAmC,CAAC;QAC/G,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mGAAmG,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzI,CAAC;IAED,QAAQ,CAAC,OAAmB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,0BAAkB,EAAE,yBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAClH,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC;IAED,SAAS,CACL,KAAa,EACb,MAA8C,EAC9C,MAA4E;QAE5E,IAAI,CAAC,EAAE;aACF,OAAO,CAAC,oIAAoI,CAAC;aAC7I,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACtG,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAa;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAA+E,CAAC;QACrH,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,kBAAkB,KAAK,0BAAkB,CAAC;IACrG,CAAC;IAED,iDAAiD;IACjD,UAAU,CAAC,IAAwB,EAAE,KAAa;QAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;gBACtB,IAAI,CAAC,cAAc,CAAC,GAAG,CACnB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,UAAU,EACf,GAAG,EACH,KAAK,EACL,0BAAkB,CACrB,CAAC;gBACF,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACtB,IAAI,CAAC,IAAI,EACT,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvB,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnC,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvC,CAAC;YACN,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAW,EAAE,GAAG,MAAkC;QAClD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,sEAAsE;IACtE,WAAW,CAAC,EAAc;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,EAAE,EAAE,CAAC;YACL,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,KAAK,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACjF,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAQ,CAAC;IACtD,CAAC;IAED,QAAQ,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACpF,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAO,IAAI,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK;QACD,IAAI,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACL,oEAAoE;QACxE,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACJ;AArPD,kCAqPC"}
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/corpus/store.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;;AAEH,sDAAyB;AAEzB,0DAA6B;AAC7B,oCAAiE;AACjE,uCAAuC;AACvC,6CAA0C;AA0B1C;;GAEG;AACH,SAAS,gBAAgB;IACrB,IAAI,CAAC;QACD,iEAAiE;QACjE,OAAQ,OAAO,CAAC,aAAa,CAA+C,CAAC,YAAY,CAAC;IAC9F,CAAC;IAAC,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CACX,yFAAyF,OAAO,CAAC,OAAO,IAAI;YACxG,kGAAkG,CACzG,CAAC;IACN,CAAC;AACL,CAAC;AAED,4EAA4E;AAC5E,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,mGAAmG;AACnG,MAAM,cAAc,GAAG,CAAC,CAAC;AAYzB,MAAa,WAAW;IACH,EAAE,CAAmB;IACrB,cAAc,CAAC;IACf,iBAAiB,CAAC;IAClB,QAAQ,CAAC;IAE1B,8EAA8E;IACrE,IAAI,CAAS;IAEtB,YAAY,IAAY;QACpB,MAAM,GAAG,GAAG,mBAAI,CAAC,OAAO,CAAC,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,iBAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;QACxC,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzC,4FAA4F;QAC5F,4FAA4F;QAC5F,0FAA0F;QAC1F,4CAA4C;QAC5C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,eAAe,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEpC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;SAQrC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA4BxC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kEAAkE,CAAC,CAAC;IACxG,CAAC;IAEO,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,CAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAgC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC;QACvH,KAAK,MAAM,SAAS,IAAI,uBAAU,EAAE,CAAC;YACjC,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC;gBAC/B,SAAS;YACb,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,IAAI,CAAC;gBACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,IAAI,UAAU,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACL,CAAC;IACL,CAAC;IAEO,gBAAgB;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAmC,CAAC;QAClH,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC;YACb,OAAO,GAAG,CAAC,KAAK,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,cAAO,GAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjF,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,GAAW;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAmC,CAAC;QAC/G,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,GAAW,EAAE,KAAa;QAC9B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mGAAmG,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzI,CAAC;IAED,QAAQ,CAAC,OAAmB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,0BAAkB,EAAE,yBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAClH,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC;IAED,SAAS,CAAC,KAAa,EAAE,MAAiB,EAAE,MAA4E;QACpH,IAAI,CAAC,EAAE;aACF,OAAO,CAAC,oIAAoI,CAAC;aAC7I,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACtG,CAAC;IAED;;;;;;OAMG;IACH,kBAAkB;QACd,OAAO,IAAI,CAAC,QAAQ,CAA+B,+EAA+E,CAAC,CAAC;IACxI,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAa;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAA+E,CAAC;QACrH,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,kBAAkB,KAAK,0BAAkB,CAAC;IACrG,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAAwB,EAAE,KAAa;QAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,SAAS,CAAC;YACN,IAAI,CAAC;gBACD,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACjC,OAAO;YACX,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,qFAAqF;gBACrF,kEAAkE;gBAClE,IAAI,OAAO,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7D,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAEO,cAAc,CAAC,IAAwB,EAAE,KAAa;QAC1D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;gBACtB,IAAI,CAAC,cAAc,CAAC,GAAG,CACnB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,UAAU,EACf,GAAG,EACH,KAAK,EACL,0BAAkB,CACrB,CAAC;gBACF,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CACtB,IAAI,CAAC,IAAI,EACT,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACrB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxB,MAAM,CAAC,aAAa,EACpB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,eAAe,EACtB,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvB,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnC,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvC,CAAC;YACN,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAW,EAAE,GAAG,MAAkC;QAClD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,sEAAsE;IACtE,WAAW,CAAC,EAAc;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,IAAI,CAAC;YACD,EAAE,EAAE,CAAC;YACL,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,KAAK,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACjF,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAQ,CAAC;IACtD,CAAC;IAED,QAAQ,CAA8B,GAAW,EAAE,GAAG,MAAkC;QACpF,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAO,IAAI,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK;QACD,IAAI,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACL,oEAAoE;QACxE,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACJ;AA1RD,kCA0RC"}
@@ -36,6 +36,35 @@ export interface PatternSets {
36
36
  /** Ids of patterns that failed to compile, for diagnostics. */
37
37
  invalid: string[];
38
38
  }
39
+ /**
40
+ * A classifier asset is missing or corrupt.
41
+ *
42
+ * An installation defect, not malformed email, and therefore deliberately outside the per message
43
+ * fail-soft path. Without the pattern files every detector returns nothing: no reply header is
44
+ * recognised, no quoted history is found, and every message comes back looking like new content.
45
+ * That reads as success, which makes it the worst possible failure mode for a package whose whole
46
+ * job is to know which parts of a message are history.
47
+ */
48
+ export declare class PatternAssetError extends Error {
49
+ readonly code = "pattern_assets_unusable";
50
+ readonly file: string | null;
51
+ constructor(message: string, file?: string | null);
52
+ }
53
+ /** Total compiled patterns across every set. */
54
+ export declare function countPatterns(sets: PatternSets): number;
55
+ /**
56
+ * Fail loudly when no pattern survived loading.
57
+ *
58
+ * The bundled patterns are read from disk at runtime through path.join(__dirname, 'patterns').
59
+ * @yao-pkg/pkg detects that form and bundles the directory automatically, which is why consumers
60
+ * need no pkg asset configuration, but it is a static analysis heuristic rather than a guarantee.
61
+ * If it ever fails, or the files are missing from a published tarball, every detector quietly
62
+ * returns nothing: no reply header is recognised and reply history stops being detected.
63
+ *
64
+ * The failure direction is at least the safe one, since nothing can be hidden without an anchor.
65
+ * But a build that silently extracts nothing useful should not pass unnoticed.
66
+ */
67
+ export declare function assertPatternsLoaded(sets: PatternSets, source: string): void;
39
68
  /** Load and compile all pattern sets. Cached after the first call. */
40
69
  export declare function loadPatterns(overrideDir?: string | null): PatternSets;
41
70
  /** Reset the cache. Used by tests that swap pattern directories. */
@@ -1 +1 @@
1
- {"version":3,"file":"patterns.d.ts","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,kBAAkB,EAAE,WAAW,EAAE,CAAC;IAClC,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACnC,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAChC,iBAAiB,EAAE,WAAW,EAAE,CAAC;IACjC,aAAa,EAAE,eAAe,EAAE,CAAC;IACjC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;AAwFD,sEAAsE;AACtE,wBAAgB,YAAY,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,WAAW,CA6BrE;AAED,oEAAoE;AACpE,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
1
+ {"version":3,"file":"patterns.d.ts","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;IAC1F,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,kBAAkB,EAAE,WAAW,EAAE,CAAC;IAClC,mBAAmB,EAAE,WAAW,EAAE,CAAC;IACnC,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAChC,iBAAiB,EAAE,WAAW,EAAE,CAAC;IACjC,aAAa,EAAE,eAAe,EAAE,CAAC;IACjC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,EAAE,CAAC;CACrB;AAcD;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IACxC,QAAQ,CAAC,IAAI,6BAA6B;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEjB,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,GAAG,IAAW;CAK1D;AAiGD,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAUvD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAQ5E;AAID,sEAAsE;AACtE,wBAAgB,YAAY,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,WAAW,CA+BrE;AAED,oEAAoE;AACpE,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
@@ -14,29 +14,73 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
14
14
  return (mod && mod.__esModule) ? mod : { "default": mod };
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.PatternAssetError = void 0;
18
+ exports.countPatterns = countPatterns;
19
+ exports.assertPatternsLoaded = assertPatternsLoaded;
17
20
  exports.loadPatterns = loadPatterns;
18
21
  exports.resetPatternCache = resetPatternCache;
19
22
  const node_fs_1 = __importDefault(require("node:fs"));
20
23
  const node_path_1 = __importDefault(require("node:path"));
21
24
  const BUNDLED_DIR = node_path_1.default.join(__dirname, 'patterns');
25
+ /**
26
+ * A classifier asset is missing or corrupt.
27
+ *
28
+ * An installation defect, not malformed email, and therefore deliberately outside the per message
29
+ * fail-soft path. Without the pattern files every detector returns nothing: no reply header is
30
+ * recognised, no quoted history is found, and every message comes back looking like new content.
31
+ * That reads as success, which makes it the worst possible failure mode for a package whose whole
32
+ * job is to know which parts of a message are history.
33
+ */
34
+ class PatternAssetError extends Error {
35
+ code = 'pattern_assets_unusable';
36
+ file;
37
+ constructor(message, file = null) {
38
+ super(message);
39
+ this.name = 'PatternAssetError';
40
+ this.file = file;
41
+ }
42
+ }
43
+ exports.PatternAssetError = PatternAssetError;
22
44
  function readJson(dir, file) {
45
+ let raw;
46
+ try {
47
+ raw = node_fs_1.default.readFileSync(node_path_1.default.join(dir, file), 'utf8');
48
+ }
49
+ catch {
50
+ return { ok: false, reason: 'missing' };
51
+ }
52
+ // Present but unreadable is a different fault from absent, and must not be silently treated as
53
+ // an empty pattern set. A truncated file in a published tarball produced exactly that.
23
54
  try {
24
- const raw = node_fs_1.default.readFileSync(node_path_1.default.join(dir, file), 'utf8');
25
55
  const parsed = JSON.parse(raw);
26
- return parsed && typeof parsed === 'object' ? parsed : null;
56
+ if (!parsed || typeof parsed !== 'object') {
57
+ return { ok: false, reason: 'invalid' };
58
+ }
59
+ return { ok: true, value: parsed };
27
60
  }
28
61
  catch {
29
- return null;
62
+ return { ok: false, reason: 'invalid' };
30
63
  }
31
64
  }
32
65
  function loadFile(file, overrideDir) {
33
66
  if (overrideDir) {
34
67
  const override = readJson(overrideDir, file);
35
- if (override) {
36
- return override;
68
+ if (override.ok) {
69
+ return override.value;
37
70
  }
71
+ if (override.reason === 'invalid') {
72
+ throw new PatternAssetError(`Pattern file ${file} in ${overrideDir} is not valid JSON.`, file);
73
+ }
74
+ // Absent from an override directory is normal: an override may replace only some files.
75
+ }
76
+ const bundled = readJson(BUNDLED_DIR, file);
77
+ if (bundled.ok) {
78
+ return bundled.value;
79
+ }
80
+ if (bundled.reason === 'invalid') {
81
+ throw new PatternAssetError(`Bundled pattern file ${file} is corrupt. Reinstall the package.`, file);
38
82
  }
39
- return readJson(BUNDLED_DIR, file) || {};
83
+ return {};
40
84
  }
41
85
  function compileLinePatterns(input, invalid) {
42
86
  if (!Array.isArray(input)) {
@@ -89,6 +133,35 @@ function compileSelectors(input, invalid) {
89
133
  }
90
134
  return out;
91
135
  }
136
+ /** Total compiled patterns across every set. */
137
+ function countPatterns(sets) {
138
+ return (sets.replyHeaders.length +
139
+ sets.forwardHeaders.length +
140
+ sets.quotedHeaderFields.length +
141
+ sets.signatureDelimiters.length +
142
+ sets.signaturePhrases.length +
143
+ sets.disclaimerPhrases.length +
144
+ sets.htmlSelectors.length);
145
+ }
146
+ /**
147
+ * Fail loudly when no pattern survived loading.
148
+ *
149
+ * The bundled patterns are read from disk at runtime through path.join(__dirname, 'patterns').
150
+ * @yao-pkg/pkg detects that form and bundles the directory automatically, which is why consumers
151
+ * need no pkg asset configuration, but it is a static analysis heuristic rather than a guarantee.
152
+ * If it ever fails, or the files are missing from a published tarball, every detector quietly
153
+ * returns nothing: no reply header is recognised and reply history stops being detected.
154
+ *
155
+ * The failure direction is at least the safe one, since nothing can be hidden without an anchor.
156
+ * But a build that silently extracts nothing useful should not pass unnoticed.
157
+ */
158
+ function assertPatternsLoaded(sets, source) {
159
+ if (countPatterns(sets) === 0) {
160
+ throw new PatternAssetError(`No detector patterns could be loaded from ${source}. ` +
161
+ 'The pattern JSON files are missing from the installation or the bundle. ' +
162
+ 'Without them no reply history can be detected.');
163
+ }
164
+ }
92
165
  let cached = null;
93
166
  /** Load and compile all pattern sets. Cached after the first call. */
94
167
  function loadPatterns(overrideDir) {
@@ -113,6 +186,7 @@ function loadPatterns(overrideDir) {
113
186
  htmlSelectors: compileSelectors(html.selectors, invalid),
114
187
  invalid
115
188
  };
189
+ assertPatternsLoaded(sets, dir ?? BUNDLED_DIR);
116
190
  if (overrideDir === undefined) {
117
191
  cached = sets;
118
192
  }
@@ -1 +1 @@
1
- {"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;AA0HH,oCA6BC;AAGD,8CAEC;AA1JD,sDAAyB;AACzB,0DAA6B;AA0C7B,MAAM,WAAW,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAErD,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAY;IACvC,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAE,MAAkC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7F,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,WAA0B;IACtD,IAAI,WAAW,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACX,OAAO,QAAQ,CAAC;QACpB,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,OAAiB;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAqB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,GAAG,CAAC,IAAI,CAAC;gBACL,EAAE;gBACF,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACjE,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBACjE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9E,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,OAAiB;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAkC,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC;YACL,EAAE;YACF,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;YAC3D,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAA4B;YACrF,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YACjE,QAAQ;YACR,UAAU,EAAE,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,sEAAsE;AACtE,SAAgB,YAAY,CAAC,WAA2B;IACpD,IAAI,MAAM,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;IACjG,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,QAAQ,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAgB;QACtB,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC1D,cAAc,EAAE,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC9D,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QAC/D,mBAAmB,EAAE,mBAAmB,CAAC,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC;QACxE,gBAAgB,EAAE,mBAAmB,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;QAClE,iBAAiB,EAAE,mBAAmB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;QACpE,aAAa,EAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;QACxD,OAAO;KACV,CAAC;IAEF,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,oEAAoE;AACpE,SAAgB,iBAAiB;IAC7B,MAAM,GAAG,IAAI,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;AAmKH,sCAUC;AAcD,oDAQC;AAKD,oCA+BC;AAGD,8CAEC;AA1OD,sDAAyB;AACzB,0DAA6B;AA0C7B,MAAM,WAAW,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAa,iBAAkB,SAAQ,KAAK;IAC/B,IAAI,GAAG,yBAAyB,CAAC;IACjC,IAAI,CAAgB;IAE7B,YAAY,OAAe,EAAE,OAAsB,IAAI;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AATD,8CASC;AAID,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IACD,+FAA+F;IAC/F,uFAAuF;IACvF,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAiC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,WAA0B;IACtD,IAAI,WAAW,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC,KAAK,CAAC;QAC1B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,iBAAiB,CAAC,gBAAgB,IAAI,OAAO,WAAW,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACnG,CAAC;QACD,wFAAwF;IAC5F,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,OAAO,CAAC,KAAK,CAAC;IACzB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,iBAAiB,CAAC,wBAAwB,IAAI,qCAAqC,EAAE,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,OAAiB;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAqB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,GAAG,CAAC,IAAI,CAAC;gBACL,EAAE;gBACF,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACjE,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBACjE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9E,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,OAAiB;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAkC,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC;YACL,EAAE;YACF,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;YAC3D,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAA4B;YACrF,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YACjE,QAAQ;YACR,UAAU,EAAE,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAgB,aAAa,CAAC,IAAiB;IAC3C,OAAO,CACH,IAAI,CAAC,YAAY,CAAC,MAAM;QACxB,IAAI,CAAC,cAAc,CAAC,MAAM;QAC1B,IAAI,CAAC,kBAAkB,CAAC,MAAM;QAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM;QAC/B,IAAI,CAAC,gBAAgB,CAAC,MAAM;QAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5B,CAAC;AACN,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,oBAAoB,CAAC,IAAiB,EAAE,MAAc;IAClE,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,iBAAiB,CACvB,6CAA6C,MAAM,IAAI;YACnD,0EAA0E;YAC1E,gDAAgD,CACvD,CAAC;IACN,CAAC;AACL,CAAC;AAED,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,sEAAsE;AACtE,SAAgB,YAAY,CAAC,WAA2B;IACpD,IAAI,MAAM,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;IACjG,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,QAAQ,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAgB;QACtB,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC1D,cAAc,EAAE,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC9D,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QAC/D,mBAAmB,EAAE,mBAAmB,CAAC,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC;QACxE,gBAAgB,EAAE,mBAAmB,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;QAClE,iBAAiB,EAAE,mBAAmB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;QACpE,aAAa,EAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;QACxD,OAAO;KACV,CAAC;IAEF,oBAAoB,CAAC,IAAI,EAAE,GAAG,IAAI,WAAW,CAAC,CAAC;IAE/C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,oEAAoE;AACpE,SAAgB,iBAAiB;IAC7B,MAAM,GAAG,IAAI,CAAC;AAClB,CAAC"}
@@ -32,6 +32,20 @@ export declare function detectAttributionLine(line: string): Evidence | null;
32
32
  *
33
33
  * Returns the first match found, scanning forward. The caller supplies de-prefixed line content so
34
34
  * that a header block inside a quote is detected the same way as one outside it.
35
+ *
36
+ * Single pass, deliberately. The obvious implementation restarts an inner forward scan at every
37
+ * candidate line, which is quadratic: 12,000 consecutive `Cc:` lines never form a strong block but
38
+ * make the inner scan traverse almost the whole remaining input from every outer position, and
39
+ * measured at 4.8 seconds of blocked event loop on that input. Inbound mail is attacker controlled,
40
+ * so that is a denial of service, not a slow path.
41
+ *
42
+ * The single pass is exactly equivalent, not an approximation. Lines partition into *chains*:
43
+ * maximal runs of header-shaped lines joined by blank gaps of at most `MAX_GAP`, terminated by any
44
+ * other line or by a longer blank run. The set of roles collected from a start inside a chain is a
45
+ * subset of the set collected from the chain's own start, and strength is monotone under superset,
46
+ * so if the chain start is not strong no later start in that chain can be, and if it is strong it
47
+ * is also the earliest strong start. Checking each chain once therefore returns the same match the
48
+ * restarting scan did.
35
49
  */
36
50
  export declare function detectQuotedHeaderBlock(lines: readonly string[], from?: number): ReplyHeaderMatch | null;
37
51
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AA4BjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAgBnE;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,SAAI,GAAG,gBAAgB,GAAG,IAAI,CAmDnG;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CASjE"}
1
+ {"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AA4BjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAgBnE;AASD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,SAAI,GAAG,gBAAgB,GAAG,IAAI,CA8EnG;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CASjE"}
@@ -61,57 +61,100 @@ function detectAttributionLine(line) {
61
61
  }
62
62
  return best;
63
63
  }
64
+ /** Is this run of header-shaped lines convincing enough to anchor a region? */
65
+ function isStrong(roles) {
66
+ // A convincing block names the originator plus at least a date or a subject, so that a
67
+ // pasted "To: ... Cc: ..." address list on its own does not qualify.
68
+ return roles.size >= MIN_FIELDS && roles.has('originator') && (roles.has('date') || roles.has('subject'));
69
+ }
64
70
  /**
65
71
  * Detect an embedded header block starting at or after `from`.
66
72
  *
67
73
  * Returns the first match found, scanning forward. The caller supplies de-prefixed line content so
68
74
  * that a header block inside a quote is detected the same way as one outside it.
75
+ *
76
+ * Single pass, deliberately. The obvious implementation restarts an inner forward scan at every
77
+ * candidate line, which is quadratic: 12,000 consecutive `Cc:` lines never form a strong block but
78
+ * make the inner scan traverse almost the whole remaining input from every outer position, and
79
+ * measured at 4.8 seconds of blocked event loop on that input. Inbound mail is attacker controlled,
80
+ * so that is a denial of service, not a slow path.
81
+ *
82
+ * The single pass is exactly equivalent, not an approximation. Lines partition into *chains*:
83
+ * maximal runs of header-shaped lines joined by blank gaps of at most `MAX_GAP`, terminated by any
84
+ * other line or by a longer blank run. The set of roles collected from a start inside a chain is a
85
+ * subset of the set collected from the chain's own start, and strength is monotone under superset,
86
+ * so if the chain start is not strong no later start in that chain can be, and if it is strong it
87
+ * is also the earliest strong start. Checking each chain once therefore returns the same match the
88
+ * restarting scan did.
69
89
  */
70
90
  function detectQuotedHeaderBlock(lines, from = 0) {
71
91
  const patterns = (0, patterns_1.loadPatterns)();
72
- for (let i = from; i < lines.length; i++) {
73
- const roles = new Set();
74
- let gap = 0;
75
- let last = -1;
76
- for (let j = i; j < lines.length; j++) {
77
- const line = (lines[j] ?? '').trim();
78
- if (!line) {
79
- gap++;
80
- if (gap > MAX_GAP) {
81
- break;
82
- }
92
+ const evidence = {
93
+ id: 'reply:quoted_header_block',
94
+ kind: 'reply-anchor',
95
+ // Below a single line attribution on purpose: a person can paste this shape.
96
+ strength: 0.88
97
+ };
98
+ // Start of the chain currently being accumulated, which may be a blank line: the restarting
99
+ // scan reported the outer index it began at, and that index is allowed to sit on the at most
100
+ // MAX_GAP blank lines that lead into the first field.
101
+ let chainStart = -1;
102
+ let roles = new Set();
103
+ let last = -1;
104
+ let gap = 0;
105
+ for (let j = Math.max(0, from); j < lines.length; j++) {
106
+ const line = (lines[j] ?? '').trim();
107
+ if (!line) {
108
+ if (chainStart === -1) {
109
+ chainStart = j;
110
+ gap = 1;
83
111
  continue;
84
112
  }
85
- const matched = patterns.quotedHeaderFields.find(pattern => pattern.regex.test(line));
86
- if (!matched) {
87
- break;
113
+ gap++;
114
+ if (gap <= MAX_GAP) {
115
+ continue;
88
116
  }
89
- gap = 0;
90
- const role = matched.role || matched.id;
91
- const value = valueOf(line);
92
- // Only fields whose value looks like a real header value count towards the threshold.
93
- // The line still continues the block either way, so a "Subject:" with unusual content
94
- // does not truncate an otherwise convincing header block.
95
- if (role === 'originator' ? ORIGINATOR_VALUE.test(value) : role === 'date' ? DATE_VALUE.test(value) : true) {
96
- roles.add(role);
117
+ // The blank run is now too long to be an internal gap. Close the chain, then begin a
118
+ // new tentative one far enough into the blank run that exactly MAX_GAP blanks can still
119
+ // precede a following field line, which is where the restarting scan would have found
120
+ // its next viable start.
121
+ if (isStrong(roles) && last >= chainStart) {
122
+ return { startLine: chainStart, endLine: last, evidence };
123
+ }
124
+ roles = new Set();
125
+ last = -1;
126
+ chainStart = j - MAX_GAP + 1;
127
+ gap = MAX_GAP;
128
+ continue;
129
+ }
130
+ const matched = patterns.quotedHeaderFields.find(pattern => pattern.regex.test(line));
131
+ if (!matched) {
132
+ // Ordinary content terminates the chain outright.
133
+ if (isStrong(roles) && last >= chainStart && chainStart !== -1) {
134
+ return { startLine: chainStart, endLine: last, evidence };
97
135
  }
98
- last = j;
136
+ roles = new Set();
137
+ last = -1;
138
+ chainStart = -1;
139
+ gap = 0;
140
+ continue;
141
+ }
142
+ if (chainStart === -1) {
143
+ chainStart = j;
99
144
  }
100
- // A convincing block names the originator plus at least a date or a subject, so that a
101
- // pasted "To: ... Cc: ..." address list on its own does not qualify.
102
- const strong = roles.size >= MIN_FIELDS && roles.has('originator') && (roles.has('date') || roles.has('subject'));
103
- if (strong && last >= i) {
104
- return {
105
- startLine: i,
106
- endLine: last,
107
- evidence: {
108
- id: 'reply:quoted_header_block',
109
- kind: 'reply-anchor',
110
- // Below a single line attribution on purpose: a person can paste this shape.
111
- strength: 0.88
112
- }
113
- };
145
+ gap = 0;
146
+ const role = matched.role || matched.id;
147
+ const value = valueOf(line);
148
+ // Only fields whose value looks like a real header value count towards the threshold.
149
+ // The line still continues the block either way, so a "Subject:" with unusual content
150
+ // does not truncate an otherwise convincing header block.
151
+ if (role === 'originator' ? ORIGINATOR_VALUE.test(value) : role === 'date' ? DATE_VALUE.test(value) : true) {
152
+ roles.add(role);
114
153
  }
154
+ last = j;
155
+ }
156
+ if (chainStart !== -1 && isStrong(roles) && last >= chainStart) {
157
+ return { startLine: chainStart, endLine: last, evidence };
115
158
  }
116
159
  return null;
117
160
  }
@@ -1 +1 @@
1
- {"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AA6CH,sDAgBC;AAQD,0DAmDC;AAQD,kDASC;AAvID,yCAA0C;AAG1C,4EAA4E;AAC5E,MAAM,OAAO,GAAG,CAAC,CAAC;AAElB,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;GAQG;AACH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF,iGAAiG;AACjG,MAAM,UAAU,GACZ,sUAAsU,CAAC;AAE3U,SAAS,OAAO,CAAC,IAAY;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5D,CAAC;AAUD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5C,IAAI,GAAG,EAAE,EAAE,EAAE,SAAS,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,uBAAuB,CAAC,KAAwB,EAAE,IAAI,GAAG,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,GAAG,EAAE,CAAC;gBACN,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;oBAChB,MAAM;gBACV,CAAC;gBACD,SAAS;YACb,CAAC;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACtF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM;YACV,CAAC;YACD,GAAG,GAAG,CAAC,CAAC;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,sFAAsF;YACtF,sFAAsF;YACtF,0DAA0D;YAC1D,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACzG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,GAAG,CAAC,CAAC;QACb,CAAC;QAED,uFAAuF;QACvF,qEAAqE;QACrE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QAClH,IAAI,MAAM,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO;gBACH,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE;oBACN,EAAE,EAAE,2BAA2B;oBAC/B,IAAI,EAAE,cAAc;oBACpB,6EAA6E;oBAC7E,QAAQ,EAAE,IAAI;iBACjB;aACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACrF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AA6CH,sDAgBC;AA6BD,0DA8EC;AAQD,kDASC;AAvLD,yCAA0C;AAG1C,4EAA4E;AAC5E,MAAM,OAAO,GAAG,CAAC,CAAC;AAElB,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;GAQG;AACH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF,iGAAiG;AACjG,MAAM,UAAU,GACZ,sUAAsU,CAAC;AAE3U,SAAS,OAAO,CAAC,IAAY;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5D,CAAC;AAUD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5C,IAAI,GAAG,EAAE,EAAE,EAAE,SAAS,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,SAAS,QAAQ,CAAC,KAA0B;IACxC,uFAAuF;IACvF,qEAAqE;IACrE,OAAO,KAAK,CAAC,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,uBAAuB,CAAC,KAAwB,EAAE,IAAI,GAAG,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAEhC,MAAM,QAAQ,GAAa;QACvB,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,cAAc;QACpB,6EAA6E;QAC7E,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,sDAAsD;IACtD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAErC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,UAAU,GAAG,CAAC,CAAC;gBACf,GAAG,GAAG,CAAC,CAAC;gBACR,SAAS;YACb,CAAC;YACD,GAAG,EAAE,CAAC;YACN,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,qFAAqF;YACrF,wFAAwF;YACxF,sFAAsF;YACtF,yBAAyB;YACzB,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;gBACxC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;YAC7B,GAAG,GAAG,OAAO,CAAC;YACd,SAAS;QACb,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,CAAC,CAAC;YAChB,GAAG,GAAG,CAAC,CAAC;YACR,SAAS;QACb,CAAC;QAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,UAAU,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,GAAG,GAAG,CAAC,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,sFAAsF;QACtF,sFAAsF;QACtF,0DAA0D;QAC1D,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;QAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9D,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACrF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}