@postalsys/email-content 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +65 -12
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/analyze.d.ts +6 -0
  4. package/dist/analyze.d.ts.map +1 -1
  5. package/dist/analyze.js +27 -6
  6. package/dist/analyze.js.map +1 -1
  7. package/dist/annotate/agreement.d.ts +99 -0
  8. package/dist/annotate/agreement.d.ts.map +1 -0
  9. package/dist/annotate/agreement.js +126 -0
  10. package/dist/annotate/agreement.js.map +1 -0
  11. package/dist/annotate/client.d.ts +7 -19
  12. package/dist/annotate/client.d.ts.map +1 -1
  13. package/dist/annotate/client.js +28 -82
  14. package/dist/annotate/client.js.map +1 -1
  15. package/dist/annotate/http.d.ts +57 -0
  16. package/dist/annotate/http.d.ts.map +1 -0
  17. package/dist/annotate/http.js +99 -0
  18. package/dist/annotate/http.js.map +1 -0
  19. package/dist/annotate/llm.d.ts +17 -0
  20. package/dist/annotate/llm.d.ts.map +1 -1
  21. package/dist/annotate/llm.js +68 -36
  22. package/dist/annotate/llm.js.map +1 -1
  23. package/dist/annotate/probabilities.d.ts +48 -0
  24. package/dist/annotate/probabilities.d.ts.map +1 -0
  25. package/dist/annotate/probabilities.js +119 -0
  26. package/dist/annotate/probabilities.js.map +1 -0
  27. package/dist/annotate/providers.d.ts +26 -0
  28. package/dist/annotate/providers.d.ts.map +1 -0
  29. package/dist/annotate/providers.js +43 -0
  30. package/dist/annotate/providers.js.map +1 -0
  31. package/dist/annotate/store.d.ts +3 -0
  32. package/dist/annotate/store.d.ts.map +1 -1
  33. package/dist/annotate/store.js +3 -2
  34. package/dist/annotate/store.js.map +1 -1
  35. package/dist/annotate/typesafe-client.d.ts +64 -0
  36. package/dist/annotate/typesafe-client.d.ts.map +1 -0
  37. package/dist/annotate/typesafe-client.js +76 -0
  38. package/dist/annotate/typesafe-client.js.map +1 -0
  39. package/dist/annotate/typesafe-prompt.d.ts +64 -0
  40. package/dist/annotate/typesafe-prompt.d.ts.map +1 -0
  41. package/dist/annotate/typesafe-prompt.js +170 -0
  42. package/dist/annotate/typesafe-prompt.js.map +1 -0
  43. package/dist/canonical/text.d.ts +15 -3
  44. package/dist/canonical/text.d.ts.map +1 -1
  45. package/dist/canonical/text.js +20 -4
  46. package/dist/canonical/text.js.map +1 -1
  47. package/dist/classify/rules.d.ts +7 -0
  48. package/dist/classify/rules.d.ts.map +1 -1
  49. package/dist/classify/rules.js +67 -14
  50. package/dist/classify/rules.js.map +1 -1
  51. package/dist/cli/agreement.d.ts +27 -0
  52. package/dist/cli/agreement.d.ts.map +1 -0
  53. package/dist/cli/agreement.js +153 -0
  54. package/dist/cli/agreement.js.map +1 -0
  55. package/dist/cli/annotate.d.ts.map +1 -1
  56. package/dist/cli/annotate.js +28 -16
  57. package/dist/cli/annotate.js.map +1 -1
  58. package/dist/cli/audit.d.ts +12 -0
  59. package/dist/cli/audit.d.ts.map +1 -0
  60. package/dist/cli/audit.js +134 -0
  61. package/dist/cli/audit.js.map +1 -0
  62. package/dist/cli/disagreements.d.ts.map +1 -1
  63. package/dist/cli/disagreements.js +20 -3
  64. package/dist/cli/disagreements.js.map +1 -1
  65. package/dist/cli/evaluate.d.ts.map +1 -1
  66. package/dist/cli/evaluate.js +5 -70
  67. package/dist/cli/evaluate.js.map +1 -1
  68. package/dist/cli/main.js +28 -2
  69. package/dist/cli/main.js.map +1 -1
  70. package/dist/corpus/analyze.d.ts.map +1 -1
  71. package/dist/corpus/analyze.js +4 -5
  72. package/dist/corpus/analyze.js.map +1 -1
  73. package/dist/corpus/migrations/index.d.ts.map +1 -1
  74. package/dist/corpus/migrations/index.js +14 -0
  75. package/dist/corpus/migrations/index.js.map +1 -1
  76. package/dist/corpus/stats.d.ts +1 -0
  77. package/dist/corpus/stats.d.ts.map +1 -1
  78. package/dist/corpus/stats.js +1 -0
  79. package/dist/corpus/stats.js.map +1 -1
  80. package/dist/detect/patterns/forward-headers.json +24 -0
  81. package/dist/detect/patterns/html-clients.json +27 -0
  82. package/dist/detect/patterns/reply-headers.json +368 -0
  83. package/dist/detect/patterns.d.ts +19 -0
  84. package/dist/detect/patterns.d.ts.map +1 -1
  85. package/dist/detect/patterns.js +53 -25
  86. package/dist/detect/patterns.js.map +1 -1
  87. package/dist/detect/reply-header.d.ts.map +1 -1
  88. package/dist/detect/reply-header.js +12 -1
  89. package/dist/detect/reply-header.js.map +1 -1
  90. package/dist/detect/subject.d.ts +16 -0
  91. package/dist/detect/subject.d.ts.map +1 -0
  92. package/dist/detect/subject.js +22 -0
  93. package/dist/detect/subject.js.map +1 -0
  94. package/dist/eval/pattern-audit.d.ts +56 -0
  95. package/dist/eval/pattern-audit.d.ts.map +1 -0
  96. package/dist/eval/pattern-audit.js +111 -0
  97. package/dist/eval/pattern-audit.js.map +1 -0
  98. package/dist/eval/reference.d.ts +44 -0
  99. package/dist/eval/reference.d.ts.map +1 -0
  100. package/dist/eval/reference.js +87 -0
  101. package/dist/eval/reference.js.map +1 -0
  102. package/dist/index.d.ts +5 -1
  103. package/dist/index.d.ts.map +1 -1
  104. package/dist/index.js +7 -2
  105. package/dist/index.js.map +1 -1
  106. package/dist/mime/flowed.d.ts +14 -0
  107. package/dist/mime/flowed.d.ts.map +1 -0
  108. package/dist/mime/flowed.js +71 -0
  109. package/dist/mime/flowed.js.map +1 -0
  110. package/dist/mime/parse.js +2 -5
  111. package/dist/mime/parse.js.map +1 -1
  112. package/dist/segment/html-blocks.d.ts.map +1 -1
  113. package/dist/segment/html-blocks.js +11 -7
  114. package/dist/segment/html-blocks.js.map +1 -1
  115. package/dist/segment/text-blocks.d.ts.map +1 -1
  116. package/dist/segment/text-blocks.js +14 -3
  117. package/dist/segment/text-blocks.js.map +1 -1
  118. package/dist/types.d.ts +18 -3
  119. package/dist/types.d.ts.map +1 -1
  120. package/dist/types.js +2 -2
  121. package/package.json +8 -8
@@ -18,6 +18,7 @@ exports.PatternAssetError = void 0;
18
18
  exports.countPatterns = countPatterns;
19
19
  exports.assertPatternsLoaded = assertPatternsLoaded;
20
20
  exports.loadPatterns = loadPatterns;
21
+ exports.patternSource = patternSource;
21
22
  exports.resetPatternCache = resetPatternCache;
22
23
  const node_fs_1 = __importDefault(require("node:fs"));
23
24
  const node_path_1 = __importDefault(require("node:path"));
@@ -66,7 +67,7 @@ function loadFile(file, overrideDir) {
66
67
  if (overrideDir) {
67
68
  const override = readJson(overrideDir, file);
68
69
  if (override.ok) {
69
- return override.value;
70
+ return { value: override.value, overridden: true };
70
71
  }
71
72
  if (override.reason === 'invalid') {
72
73
  throw new PatternAssetError(`Pattern file ${file} in ${overrideDir} is not valid JSON.`, file);
@@ -75,12 +76,12 @@ function loadFile(file, overrideDir) {
75
76
  }
76
77
  const bundled = readJson(BUNDLED_DIR, file);
77
78
  if (bundled.ok) {
78
- return bundled.value;
79
+ return { value: bundled.value, overridden: false };
79
80
  }
80
81
  if (bundled.reason === 'invalid') {
81
82
  throw new PatternAssetError(`Bundled pattern file ${file} is corrupt. Reinstall the package.`, file);
82
83
  }
83
- return {};
84
+ return { value: {}, overridden: false };
84
85
  }
85
86
  function compileLinePatterns(input, invalid) {
86
87
  if (!Array.isArray(input)) {
@@ -171,38 +172,65 @@ function assertPatternsLoaded(sets, source) {
171
172
  'Without them no reply history can be detected.');
172
173
  }
173
174
  }
174
- let cached = null;
175
- /** Load and compile all pattern sets. Cached after the first call. */
176
- function loadPatterns(overrideDir) {
177
- if (cached && overrideDir === undefined) {
178
- return cached;
179
- }
180
- const dir = overrideDir === undefined ? process.env.EMAIL_CONTENT_PATTERNS || null : overrideDir;
175
+ const PATTERN_FILES = [
176
+ 'reply-headers.json',
177
+ 'forward-headers.json',
178
+ 'quoted-headers.json',
179
+ 'signatures.json',
180
+ 'disclaimers.json',
181
+ 'html-clients.json'
182
+ ];
183
+ // The compiled sets and the source they came from are cached together, so neither can be read
184
+ // without the other and a reset clears both at once.
185
+ let cache = null;
186
+ function loadUncached(dir) {
181
187
  const invalid = [];
182
- const reply = loadFile('reply-headers.json', dir);
183
- const forward = loadFile('forward-headers.json', dir);
184
- const quoted = loadFile('quoted-headers.json', dir);
185
- const signatures = loadFile('signatures.json', dir);
186
- const disclaimers = loadFile('disclaimers.json', dir);
187
- const html = loadFile('html-clients.json', dir);
188
+ const overriddenFiles = [];
189
+ const files = {};
190
+ for (const file of PATTERN_FILES) {
191
+ const loaded = loadFile(file, dir);
192
+ files[file] = loaded.value;
193
+ if (loaded.overridden) {
194
+ overriddenFiles.push(file);
195
+ }
196
+ }
197
+ const signatures = files['signatures.json'];
188
198
  const sets = {
189
- replyHeaders: compileLinePatterns(reply.patterns, invalid),
190
- forwardHeaders: compileLinePatterns(forward.patterns, invalid),
191
- quotedHeaderFields: compileLinePatterns(quoted.fields, invalid),
199
+ replyHeaders: compileLinePatterns(files['reply-headers.json'].patterns, invalid),
200
+ forwardHeaders: compileLinePatterns(files['forward-headers.json'].patterns, invalid),
201
+ quotedHeaderFields: compileLinePatterns(files['quoted-headers.json'].fields, invalid),
192
202
  signatureDelimiters: compileLinePatterns(signatures.delimiters, invalid),
193
203
  signaturePhrases: compileLinePatterns(signatures.phrases, invalid),
194
- disclaimerPhrases: compileLinePatterns(disclaimers.phrases, invalid),
195
- htmlSelectors: compileSelectors(html.selectors, invalid),
204
+ disclaimerPhrases: compileLinePatterns(files['disclaimers.json'].phrases, invalid),
205
+ htmlSelectors: compileSelectors(files['html-clients.json'].selectors, invalid),
196
206
  invalid
197
207
  };
198
208
  assertPatternsLoaded(sets, dir ?? BUNDLED_DIR);
199
- if (overrideDir === undefined) {
200
- cached = sets;
209
+ return { sets, source: { override: dir, overriddenFiles } };
210
+ }
211
+ function loadCached() {
212
+ if (!cache) {
213
+ cache = loadUncached(process.env.EMAIL_CONTENT_PATTERNS || null);
201
214
  }
202
- return sets;
215
+ return cache;
216
+ }
217
+ /** Load and compile all pattern sets. Cached after the first call. */
218
+ function loadPatterns(overrideDir) {
219
+ return overrideDir === undefined ? loadCached().sets : loadUncached(overrideDir).sets;
220
+ }
221
+ /**
222
+ * Report which pattern source is in effect, so a caller can log it at startup.
223
+ *
224
+ * The override is read from an unprefixed environment variable, so it can be set for a host
225
+ * process without anyone noticing. Nothing in the extraction result says the patterns differ from
226
+ * the bundled ones; this is the only way to find out.
227
+ */
228
+ function patternSource() {
229
+ const { source } = loadCached();
230
+ return { override: source.override, overriddenFiles: [...source.overriddenFiles] };
203
231
  }
204
232
  /** Reset the cache. Used by tests that swap pattern directories. */
205
233
  function resetPatternCache() {
206
- cached = null;
234
+ cache = null;
207
235
  }
208
236
  //# sourceMappingURL=patterns.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;AA4KH,sCAUC;AAcD,oDAQC;AAKD,oCA+BC;AAGD,8CAEC;AAnPD,sDAAyB;AACzB,0DAA6B;AA0C7B,MAAM,WAAW,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAa,iBAAkB,SAAQ,KAAK;IAC/B,IAAI,GAAG,yBAAyB,CAAC;IACjC,IAAI,CAAgB;IAE7B,YAAY,OAAe,EAAE,OAAsB,IAAI;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AATD,8CASC;AAID,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IACD,+FAA+F;IAC/F,uFAAuF;IACvF,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAiC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,WAA0B;IACtD,IAAI,WAAW,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,QAAQ,CAAC,KAAK,CAAC;QAC1B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,iBAAiB,CAAC,gBAAgB,IAAI,OAAO,WAAW,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACnG,CAAC;QACD,wFAAwF;IAC5F,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,OAAO,CAAC,KAAK,CAAC;IACzB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,iBAAiB,CAAC,wBAAwB,IAAI,qCAAqC,EAAE,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,OAAiB;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAqB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,yFAAyF;QACzF,yFAAyF;QACzF,yDAAyD;QACzD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;QACzE,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACnF,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,GAAG,CAAC,IAAI,CAAC;gBACL,EAAE;gBACF,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACjE,QAAQ;gBACR,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;aACnC,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,OAAiB;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAkC,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC;YACL,EAAE;YACF,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;YAC3D,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAA4B;YACrF,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YACjE,QAAQ;YACR,UAAU,EAAE,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAgB,aAAa,CAAC,IAAiB;IAC3C,OAAO,CACH,IAAI,CAAC,YAAY,CAAC,MAAM;QACxB,IAAI,CAAC,cAAc,CAAC,MAAM;QAC1B,IAAI,CAAC,kBAAkB,CAAC,MAAM;QAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM;QAC/B,IAAI,CAAC,gBAAgB,CAAC,MAAM;QAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5B,CAAC;AACN,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,oBAAoB,CAAC,IAAiB,EAAE,MAAc;IAClE,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,iBAAiB,CACvB,6CAA6C,MAAM,IAAI;YACnD,0EAA0E;YAC1E,gDAAgD,CACvD,CAAC;IACN,CAAC;AACL,CAAC;AAED,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,sEAAsE;AACtE,SAAgB,YAAY,CAAC,WAA2B;IACpD,IAAI,MAAM,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC;IACjG,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,QAAQ,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAgB;QACtB,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC1D,cAAc,EAAE,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC9D,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QAC/D,mBAAmB,EAAE,mBAAmB,CAAC,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC;QACxE,gBAAgB,EAAE,mBAAmB,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;QAClE,iBAAiB,EAAE,mBAAmB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;QACpE,aAAa,EAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC;QACxD,OAAO;KACV,CAAC;IAEF,oBAAoB,CAAC,IAAI,EAAE,GAAG,IAAI,WAAW,CAAC,CAAC;IAE/C,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,oEAAoE;AACpE,SAAgB,iBAAiB;IAC7B,MAAM,GAAG,IAAI,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/detect/patterns.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;AAiLH,sCAUC;AAcD,oDAQC;AAqDD,oCAEC;AAqBD,sCAGC;AAGD,8CAEC;AAnSD,sDAAyB;AACzB,0DAA6B;AA0C7B,MAAM,WAAW,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH,MAAa,iBAAkB,SAAQ,KAAK;IAC/B,IAAI,GAAG,yBAAyB,CAAC;IACjC,IAAI,CAAgB;IAE7B,YAAY,OAAe,EAAE,OAAsB,IAAI;QACnD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AATD,8CASC;AAID,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAY;IACvC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACD,GAAG,GAAG,iBAAE,CAAC,YAAY,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;IACD,+FAA+F;IAC/F,uFAAuF;IACvF,IAAI,CAAC;QACD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAiC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;AACL,CAAC;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,WAA0B;IACtD,IAAI,WAAW,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACvD,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,iBAAiB,CAAC,gBAAgB,IAAI,OAAO,WAAW,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACnG,CAAC;QACD,wFAAwF;IAC5F,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,iBAAiB,CAAC,wBAAwB,IAAI,qCAAqC,EAAE,IAAI,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,OAAiB;IAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAqB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,yFAAyF;QACzF,yFAAyF;QACzF,yDAAyD;QACzD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;QACzE,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACnF,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS;QACb,CAAC;QACD,IAAI,CAAC;YACD,GAAG,CAAC,IAAI,CAAC;gBACL,EAAE;gBACF,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC3D,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACjE,QAAQ;gBACR,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;aACnC,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,OAAiB;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAkC,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC;YAC3C,SAAS;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC;YACL,EAAE;YACF,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG;YAC3D,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAA4B;YACrF,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YACjE,QAAQ;YACR,UAAU,EAAE,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxE,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAgB,aAAa,CAAC,IAAiB;IAC3C,OAAO,CACH,IAAI,CAAC,YAAY,CAAC,MAAM;QACxB,IAAI,CAAC,cAAc,CAAC,MAAM;QAC1B,IAAI,CAAC,kBAAkB,CAAC,MAAM;QAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM;QAC/B,IAAI,CAAC,gBAAgB,CAAC,MAAM;QAC5B,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5B,CAAC;AACN,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,oBAAoB,CAAC,IAAiB,EAAE,MAAc;IAClE,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,iBAAiB,CACvB,6CAA6C,MAAM,IAAI;YACnD,0EAA0E;YAC1E,gDAAgD,CACvD,CAAC;IACN,CAAC;AACL,CAAC;AAED,MAAM,aAAa,GAAG;IAClB,oBAAoB;IACpB,sBAAsB;IACtB,qBAAqB;IACrB,iBAAiB;IACjB,kBAAkB;IAClB,mBAAmB;CACb,CAAC;AAIX,8FAA8F;AAC9F,qDAAqD;AACrD,IAAI,KAAK,GAAwD,IAAI,CAAC;AAEtE,SAAS,YAAY,CAAC,GAAkB;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,EAAkD,CAAC;IACjE,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QAC3B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAgB;QACtB,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC;QAChF,cAAc,EAAE,mBAAmB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC;QACpF,kBAAkB,EAAE,mBAAmB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC;QACrF,mBAAmB,EAAE,mBAAmB,CAAC,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC;QACxE,gBAAgB,EAAE,mBAAmB,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;QAClE,iBAAiB,EAAE,mBAAmB,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC;QAClF,aAAa,EAAE,gBAAgB,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC;QAC9E,OAAO;KACV,CAAC;IAEF,oBAAoB,CAAC,IAAI,EAAE,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,UAAU;IACf,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,sEAAsE;AACtE,SAAgB,YAAY,CAAC,WAA2B;IACpD,OAAO,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC;AAC1F,CAAC;AAcD;;;;;;GAMG;AACH,SAAgB,aAAa;IACzB,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;IAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;AACvF,CAAC;AAED,oEAAoE;AACpE,SAAgB,iBAAiB;IAC7B,KAAK,GAAG,IAAI,CAAC;AACjB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAkCjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAgBnE;AASD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,SAAI,GAAG,gBAAgB,GAAG,IAAI,CA8EnG;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CASjE"}
1
+ {"version":3,"file":"reply-header.d.ts","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAkCjD,MAAM,WAAW,gBAAgB;IAC7B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CA2BnE;AASD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,SAAI,GAAG,gBAAgB,GAAG,IAAI,CA8EnG;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CASjE"}
@@ -51,7 +51,18 @@ function valueOf(line) {
51
51
  * one is almost always a paragraph that happens to end in "wrote:".
52
52
  */
53
53
  function detectAttributionLine(line) {
54
- const trimmed = line.trim();
54
+ // Clients that derive plain text from HTML write links out as "mailto:" URLs, and SnappyMail
55
+ // puts one inside every attribution, fully URL encoded: '"Marta Kask" <marta@example.com
56
+ // mailto:marta@example.com?to=%22Marta...%3E > schrieb:'. The URL says nothing about the line's
57
+ // shape but pushes it past the length limits, so it is dropped before matching. The raw line is
58
+ // bounded first so the replace stays cheap on hostile input.
59
+ if (line.length > 2000) {
60
+ return null;
61
+ }
62
+ // Only lines that contain a URL pay for the replace, and it has no leading whitespace class:
63
+ // "\s*" before the literal was retried at every position of a long space run, quadratic in
64
+ // its length (1 MB of such lines blocked for over a second).
65
+ const trimmed = (/mailto:/i.test(line) ? line.replace(/ ?\bmailto:\S+/gi, '') : line).trim();
55
66
  if (!trimmed || trimmed.length > 300) {
56
67
  return null;
57
68
  }
@@ -1 +1 @@
1
- {"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AAmDH,sDAgBC;AA6BD,0DA8EC;AAQD,kDASC;AA7LD,yCAA0C;AAG1C,4EAA4E;AAC5E,MAAM,OAAO,GAAG,CAAC,CAAC;AAElB,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;GAQG;AACH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF;;;;;;GAMG;AACH,MAAM,UAAU,GACZ,wfAAwf,CAAC;AAE7f,SAAS,OAAO,CAAC,IAAY;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5D,CAAC;AAUD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5C,IAAI,GAAG,EAAE,EAAE,EAAE,SAAS,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,SAAS,QAAQ,CAAC,KAA0B;IACxC,uFAAuF;IACvF,qEAAqE;IACrE,OAAO,KAAK,CAAC,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,uBAAuB,CAAC,KAAwB,EAAE,IAAI,GAAG,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAEhC,MAAM,QAAQ,GAAa;QACvB,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,cAAc;QACpB,6EAA6E;QAC7E,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,sDAAsD;IACtD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAErC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,UAAU,GAAG,CAAC,CAAC;gBACf,GAAG,GAAG,CAAC,CAAC;gBACR,SAAS;YACb,CAAC;YACD,GAAG,EAAE,CAAC;YACN,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,qFAAqF;YACrF,wFAAwF;YACxF,sFAAsF;YACtF,yBAAyB;YACzB,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;gBACxC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;YAC7B,GAAG,GAAG,OAAO,CAAC;YACd,SAAS;QACb,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,CAAC,CAAC;YAChB,GAAG,GAAG,CAAC,CAAC;YACR,SAAS;QACb,CAAC;QAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,UAAU,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,GAAG,GAAG,CAAC,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,sFAAsF;QACtF,sFAAsF;QACtF,0DAA0D;QAC1D,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;QAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9D,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACrF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"reply-header.js","sourceRoot":"","sources":["../../src/detect/reply-header.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AAmDH,sDA2BC;AA6BD,0DA8EC;AAQD,kDASC;AAxMD,yCAA0C;AAG1C,4EAA4E;AAC5E,MAAM,OAAO,GAAG,CAAC,CAAC;AAElB,qFAAqF;AACrF,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;;;;;GAQG;AACH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF;;;;;;GAMG;AACH,MAAM,UAAU,GACZ,wfAAwf,CAAC;AAE7f,SAAS,OAAO,CAAC,IAAY;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5D,CAAC;AAUD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,IAAY;IAC9C,6FAA6F;IAC7F,yFAAyF;IACzF,gGAAgG;IAChG,gGAAgG;IAChG,6DAA6D;IAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,6FAA6F;IAC7F,2FAA2F;IAC3F,6DAA6D;IAC7D,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7F,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,IAAI,GAAoB,IAAI,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5C,IAAI,GAAG,EAAE,EAAE,EAAE,SAAS,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,SAAS,QAAQ,CAAC,KAA0B;IACxC,uFAAuF;IACvF,qEAAqE;IACrE,OAAO,KAAK,CAAC,IAAI,IAAI,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,uBAAuB,CAAC,KAAwB,EAAE,IAAI,GAAG,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAEhC,MAAM,QAAQ,GAAa;QACvB,EAAE,EAAE,2BAA2B;QAC/B,IAAI,EAAE,cAAc;QACpB,6EAA6E;QAC7E,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,sDAAsD;IACtD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAErC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,UAAU,GAAG,CAAC,CAAC;gBACf,GAAG,GAAG,CAAC,CAAC;gBACR,SAAS;YACb,CAAC;YACD,GAAG,EAAE,CAAC;YACN,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;gBACjB,SAAS;YACb,CAAC;YACD,qFAAqF;YACrF,wFAAwF;YACxF,sFAAsF;YACtF,yBAAyB;YACzB,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;gBACxC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;YAC7B,GAAG,GAAG,OAAO,CAAC;YACd,SAAS;QACb,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC9D,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;YAC1B,IAAI,GAAG,CAAC,CAAC,CAAC;YACV,UAAU,GAAG,CAAC,CAAC,CAAC;YAChB,GAAG,GAAG,CAAC,CAAC;YACR,SAAS;QACb,CAAC;QAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,UAAU,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,GAAG,GAAG,CAAC,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,sFAAsF;QACtF,sFAAsF;QACtF,0DAA0D;QAC1D,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;QAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9D,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,EAAE,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACrF,CAAC;IACD,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Subject prefixes that mark a reply or a forward, shared by analyze() and the corpus scanner so
3
+ * the two cannot drift apart again.
4
+ *
5
+ * Forward prefixes were measured on 266 real forwards from 16 clients (test/fixtures/clients):
6
+ * every one used Fwd, Fw or Ed (Outlook's Estonian prefix), whatever the interface language, and
7
+ * 250 of them also carried In-Reply-To or References, so headers cannot tell a forward from a
8
+ * reply. The other entries are the prefixes Outlook and webmail clients write in other locales.
9
+ * A reply to a forward reads "Re: Fwd:" and is caught by REPLY_SUBJECT first, which is also why
10
+ * "VS" (a Finnish reply, a Norwegian forward) is listed only as a reply.
11
+ */
12
+ export declare const REPLY_SUBJECT: RegExp;
13
+ export declare const FORWARD_SUBJECT: RegExp;
14
+ /** A forward, and not a reply to one. */
15
+ export declare function isForwardSubject(subject: string): boolean;
16
+ //# sourceMappingURL=subject.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"subject.d.ts","sourceRoot":"","sources":["../../src/detect/subject.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,eAAO,MAAM,aAAa,QAAgE,CAAC;AAE3F,eAAO,MAAM,eAAe,QAA0F,CAAC;AAEvH,yCAAyC;AACzC,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAEzD"}
@@ -0,0 +1,22 @@
1
+ "use strict";
2
+ /**
3
+ * Subject prefixes that mark a reply or a forward, shared by analyze() and the corpus scanner so
4
+ * the two cannot drift apart again.
5
+ *
6
+ * Forward prefixes were measured on 266 real forwards from 16 clients (test/fixtures/clients):
7
+ * every one used Fwd, Fw or Ed (Outlook's Estonian prefix), whatever the interface language, and
8
+ * 250 of them also carried In-Reply-To or References, so headers cannot tell a forward from a
9
+ * reply. The other entries are the prefixes Outlook and webmail clients write in other locales.
10
+ * A reply to a forward reads "Re: Fwd:" and is caught by REPLY_SUBJECT first, which is also why
11
+ * "VS" (a Finnish reply, a Norwegian forward) is listed only as a reply.
12
+ */
13
+ Object.defineProperty(exports, "__esModule", { value: true });
14
+ exports.FORWARD_SUBJECT = exports.REPLY_SUBJECT = void 0;
15
+ exports.isForwardSubject = isForwardSubject;
16
+ exports.REPLY_SUBJECT = /^\s*(re|aw|sv|vs|vast|antw|res|odp|ответ)\s*(\[\d+\])?\s*:/i;
17
+ exports.FORWARD_SUBJECT = /^\s*(fwd?|wg|tr|rv|enc|doorst|pd|vl|ed|edasi|пересл[а-я]*|转发|轉寄|転送)\s*(\[\d+\])?\s*:/i;
18
+ /** A forward, and not a reply to one. */
19
+ function isForwardSubject(subject) {
20
+ return !exports.REPLY_SUBJECT.test(subject) && exports.FORWARD_SUBJECT.test(subject);
21
+ }
22
+ //# sourceMappingURL=subject.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"subject.js","sourceRoot":"","sources":["../../src/detect/subject.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAOH,4CAEC;AAPY,QAAA,aAAa,GAAG,6DAA6D,CAAC;AAE9E,QAAA,eAAe,GAAG,uFAAuF,CAAC;AAEvH,yCAAyC;AACzC,SAAgB,gBAAgB,CAAC,OAAe;IAC5C,OAAO,CAAC,qBAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,uBAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACzE,CAAC"}
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Pattern precision audit.
3
+ *
4
+ * Aggregate agreement says how the extractor does overall; it cannot say which pattern to fix.
5
+ * This groups reference-labelled blocks by the detector evidence they carry and by the reasons
6
+ * behind their final decision, so a single pattern that fires on authored text stands out even
7
+ * when the headline numbers look fine.
8
+ *
9
+ * Two views, because they answer different questions:
10
+ *
11
+ * - Evidence: every block a pattern fired on, whatever the rules then did with it. This is the
12
+ * pattern's own precision, the thing a pattern file edit changes.
13
+ * - Reasons: every block whose final decision names a reason. This is the rule's precision, and
14
+ * for hiding reasons the rows that matter most, since a wrong hide is content loss.
15
+ *
16
+ * Evidence sits on the block that matched, usually the attribution line or separator itself, so
17
+ * the evidence view measures whether the boundary was real, not how far the region after it ran.
18
+ * The reason view covers the region.
19
+ */
20
+ import { type Label } from '../types';
21
+ /** One reference-labelled block with what the extractor saw and decided. Never carries content. */
22
+ export interface AuditBlock {
23
+ reference: Label;
24
+ predicted: Label;
25
+ hidden: boolean;
26
+ chars: number;
27
+ evidence: readonly {
28
+ id: string;
29
+ kind: string;
30
+ }[];
31
+ reasons: readonly string[];
32
+ }
33
+ export interface AuditRow {
34
+ key: string;
35
+ /** Reference labels a hit is expected to carry, or null when the kind implies none. */
36
+ expected: readonly Label[] | null;
37
+ blocks: number;
38
+ chars: number;
39
+ /** Blocks whose reference label is in `expected`. */
40
+ matching: number;
41
+ /** Blocks precision is measured over: all of them, less those `neutral` excludes. */
42
+ scored: number;
43
+ /** Blocks the reference says the sender wrote: NEW_CONTENT or INLINE_QUOTE. */
44
+ authored: number;
45
+ /** Authored blocks the extractor hid. Each one is lost content. */
46
+ authoredHidden: number;
47
+ authoredHiddenChars: number;
48
+ byReference: Partial<Record<Label, number>>;
49
+ }
50
+ export interface PatternAudit {
51
+ blocks: number;
52
+ evidence: AuditRow[];
53
+ reasons: AuditRow[];
54
+ }
55
+ export declare function auditPatterns(blocks: readonly AuditBlock[]): PatternAudit;
56
+ //# sourceMappingURL=pattern-audit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pattern-audit.d.ts","sourceRoot":"","sources":["../../src/eval/pattern-audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,UAAU,CAAC;AAEtC,mGAAmG;AACnG,MAAM,WAAW,UAAU;IACvB,SAAS,EAAE,KAAK,CAAC;IACjB,SAAS,EAAE,KAAK,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,SAAS;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAClD,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,QAAQ;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,uFAAuF;IACvF,QAAQ,EAAE,SAAS,KAAK,EAAE,GAAG,IAAI,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,OAAO,EAAE,QAAQ,EAAE,CAAC;CACvB;AAoED,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,GAAG,YAAY,CAkCzE"}
@@ -0,0 +1,111 @@
1
+ "use strict";
2
+ /**
3
+ * Pattern precision audit.
4
+ *
5
+ * Aggregate agreement says how the extractor does overall; it cannot say which pattern to fix.
6
+ * This groups reference-labelled blocks by the detector evidence they carry and by the reasons
7
+ * behind their final decision, so a single pattern that fires on authored text stands out even
8
+ * when the headline numbers look fine.
9
+ *
10
+ * Two views, because they answer different questions:
11
+ *
12
+ * - Evidence: every block a pattern fired on, whatever the rules then did with it. This is the
13
+ * pattern's own precision, the thing a pattern file edit changes.
14
+ * - Reasons: every block whose final decision names a reason. This is the rule's precision, and
15
+ * for hiding reasons the rows that matter most, since a wrong hide is content loss.
16
+ *
17
+ * Evidence sits on the block that matched, usually the attribution line or separator itself, so
18
+ * the evidence view measures whether the boundary was real, not how far the region after it ran.
19
+ * The reason view covers the region.
20
+ */
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.auditPatterns = auditPatterns;
23
+ /**
24
+ * The labels a hit of each evidence kind should land on, following docs/annotation-guide.md:
25
+ * a boundary line belongs to the region it introduces, and a forward nested inside quoted history
26
+ * is REPLY_HISTORY. A quote container legitimately holds inline quotes too. Kinds that only hint
27
+ * are reported without a precision.
28
+ */
29
+ const EXPECTED = {
30
+ 'reply-anchor': ['REPLY_HISTORY'],
31
+ attribution: ['REPLY_HISTORY'],
32
+ 'forward-anchor': ['FORWARDED_CONTENT', 'REPLY_HISTORY'],
33
+ 'quote-container': ['REPLY_HISTORY', 'INLINE_QUOTE', 'FORWARDED_CONTENT'],
34
+ signature: ['SIGNATURE'],
35
+ disclaimer: ['DISCLAIMER']
36
+ };
37
+ /**
38
+ * Reference labels that neither confirm nor refute a hit. A signature or disclaimer inside quoted
39
+ * history belongs to the history (the guide labels it by the message that attached it), so the
40
+ * pattern matching there is right about the shape and says nothing about the label. Without this,
41
+ * every sign-off pattern reads as mostly wrong simply because most sign-offs are quoted.
42
+ */
43
+ const NEUTRAL = {
44
+ signature: ['REPLY_HISTORY', 'FORWARDED_CONTENT'],
45
+ disclaimer: ['REPLY_HISTORY', 'FORWARDED_CONTENT']
46
+ };
47
+ function isAuthored(label) {
48
+ return label === 'NEW_CONTENT' || label === 'INLINE_QUOTE';
49
+ }
50
+ function emptyTally(key, expected, neutral = []) {
51
+ return {
52
+ row: { key, expected, blocks: 0, chars: 0, matching: 0, scored: 0, authored: 0, authoredHidden: 0, authoredHiddenChars: 0, byReference: {} },
53
+ neutral
54
+ };
55
+ }
56
+ function count({ row, neutral }, block) {
57
+ row.blocks++;
58
+ row.chars += block.chars;
59
+ row.byReference[block.reference] = (row.byReference[block.reference] ?? 0) + 1;
60
+ if (!neutral.includes(block.reference)) {
61
+ row.scored++;
62
+ if (row.expected?.includes(block.reference)) {
63
+ row.matching++;
64
+ }
65
+ }
66
+ if (isAuthored(block.reference)) {
67
+ row.authored++;
68
+ if (block.hidden) {
69
+ row.authoredHidden++;
70
+ row.authoredHiddenChars += block.chars;
71
+ }
72
+ }
73
+ }
74
+ /** Worst first: lost content, then the most authored hits, then volume. */
75
+ function byHarm(a, b) {
76
+ return b.authoredHiddenChars - a.authoredHiddenChars || b.authored - a.authored || b.blocks - a.blocks || a.key.localeCompare(b.key);
77
+ }
78
+ function auditPatterns(blocks) {
79
+ const evidence = new Map();
80
+ const reasons = new Map();
81
+ for (const block of blocks) {
82
+ // A pattern that matched two lines of the same block still judged one block.
83
+ const seen = new Set();
84
+ for (const item of block.evidence) {
85
+ if (seen.has(item.id)) {
86
+ continue;
87
+ }
88
+ seen.add(item.id);
89
+ let row = evidence.get(item.id);
90
+ if (!row) {
91
+ row = emptyTally(item.id, EXPECTED[item.kind] ?? null, NEUTRAL[item.kind]);
92
+ evidence.set(item.id, row);
93
+ }
94
+ count(row, block);
95
+ }
96
+ for (const reason of new Set(block.reasons)) {
97
+ // A reason is scored against the label the rules gave it: the question is whether the
98
+ // decision it justified was right.
99
+ const key = `${reason} -> ${block.predicted}`;
100
+ let row = reasons.get(key);
101
+ if (!row) {
102
+ row = emptyTally(key, [block.predicted]);
103
+ reasons.set(key, row);
104
+ }
105
+ count(row, block);
106
+ }
107
+ }
108
+ const rows = (tallies) => [...tallies.values()].map(tally => tally.row).sort(byHarm);
109
+ return { blocks: blocks.length, evidence: rows(evidence), reasons: rows(reasons) };
110
+ }
111
+ //# sourceMappingURL=pattern-audit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pattern-audit.js","sourceRoot":"","sources":["../../src/eval/pattern-audit.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;GAkBG;;AAwGH,sCAkCC;AApGD;;;;;GAKG;AACH,MAAM,QAAQ,GAAqC;IAC/C,cAAc,EAAE,CAAC,eAAe,CAAC;IACjC,WAAW,EAAE,CAAC,eAAe,CAAC;IAC9B,gBAAgB,EAAE,CAAC,mBAAmB,EAAE,eAAe,CAAC;IACxD,iBAAiB,EAAE,CAAC,eAAe,EAAE,cAAc,EAAE,mBAAmB,CAAC;IACzE,SAAS,EAAE,CAAC,WAAW,CAAC;IACxB,UAAU,EAAE,CAAC,YAAY,CAAC;CAC7B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,GAAqC;IAC9C,SAAS,EAAE,CAAC,eAAe,EAAE,mBAAmB,CAAC;IACjD,UAAU,EAAE,CAAC,eAAe,EAAE,mBAAmB,CAAC;CACrD,CAAC;AAEF,SAAS,UAAU,CAAC,KAAY;IAC5B,OAAO,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,cAAc,CAAC;AAC/D,CAAC;AAOD,SAAS,UAAU,CAAC,GAAW,EAAE,QAAiC,EAAE,UAA4B,EAAE;IAC9F,OAAO;QACH,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE;QAC5I,OAAO;KACV,CAAC;AACN,CAAC;AAED,SAAS,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,EAAS,EAAE,KAAiB;IACrD,GAAG,CAAC,MAAM,EAAE,CAAC;IACb,GAAG,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;IACzB,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,GAAG,CAAC,MAAM,EAAE,CAAC;QACb,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnB,CAAC;IACL,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACf,GAAG,CAAC,cAAc,EAAE,CAAC;YACrB,GAAG,CAAC,mBAAmB,IAAI,KAAK,CAAC,KAAK,CAAC;QAC3C,CAAC;IACL,CAAC;AACL,CAAC;AAED,2EAA2E;AAC3E,SAAS,MAAM,CAAC,CAAW,EAAE,CAAW;IACpC,OAAO,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACzI,CAAC;AAED,SAAgB,aAAa,CAAC,MAA6B;IACvD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,6EAA6E;QAC7E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,SAAS;YACb,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClB,IAAI,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC3E,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAC/B,CAAC;YACD,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1C,sFAAsF;YACtF,mCAAmC;YACnC,MAAM,GAAG,GAAG,GAAG,MAAM,OAAO,KAAK,CAAC,SAAS,EAAE,CAAC;YAC9C,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,OAA2B,EAAE,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzG,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;AACvF,CAAC"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Loading the stored LLM reference labels, shared by `evaluate` and `audit`.
3
+ *
4
+ * Both commands must read the same reference, or a pattern that looks precise in one report and
5
+ * harmful in the other is only a difference in which annotations were counted.
6
+ */
7
+ import { type CorpusStore } from '../corpus/store';
8
+ import { type Label, type Segment } from '../types';
9
+ export interface ReferenceRow {
10
+ path: string;
11
+ block_index: number;
12
+ /** MIME part the block came from. Null on annotations made before it was recorded. */
13
+ mime_part: string | null;
14
+ label: Label;
15
+ char_count: number;
16
+ client: string | null;
17
+ lang: string | null;
18
+ reply_style: string | null;
19
+ }
20
+ /**
21
+ * Latest annotation per (path, block) for the chosen model, grouped by message, so re-annotated
22
+ * messages do not count twice. An explicit prompt version pins the reference to labels made under
23
+ * one convention; without it, the newest label per block wins, whatever prompt produced it, and
24
+ * messages labelled only under an older convention are still included. The annotation guide's rule
25
+ * that runs made under different conventions must never be silently compared is the reason the
26
+ * option exists; use it whenever more than one prompt version is present.
27
+ */
28
+ export declare function loadReference(store: CorpusStore, model: string, promptVersion?: string): Map<string, ReferenceRow[]>;
29
+ export interface CoveringPrediction {
30
+ label: Label;
31
+ hidden: boolean;
32
+ }
33
+ /**
34
+ * Map a stored (part, block index) to the output segment covering it. Segments merge adjacent
35
+ * blocks, so a block is matched by its index falling inside a segment's block range.
36
+ *
37
+ * Keyed per MIME part, not by block index alone. Block indices are section-local, so in a multipart
38
+ * message with several body sections two segments legitimately start at index 0, and a flat map
39
+ * silently kept only the last of them. Older annotations predate the part being recorded; those
40
+ * fall back to the first part the output used, which is exactly the previous behaviour for the
41
+ * single section messages they came from.
42
+ */
43
+ export declare function segmentLookup(...segmentLists: readonly (readonly Segment[])[]): (part: string | null, index: number) => CoveringPrediction | null;
44
+ //# sourceMappingURL=reference.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reference.d.ts","sourceRoot":"","sources":["../../src/eval/reference.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAA4B,KAAK,KAAK,EAAE,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAE9E,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,sFAAsF;IACtF,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,KAAK,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,SAAK,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CA2BhH;AAED,MAAM,WAAW,kBAAkB;IAC/B,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAAC,GAAG,YAAY,EAAE,SAAS,CAAC,SAAS,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,KAAK,kBAAkB,GAAG,IAAI,CA+BjJ"}
@@ -0,0 +1,87 @@
1
+ "use strict";
2
+ /**
3
+ * Loading the stored LLM reference labels, shared by `evaluate` and `audit`.
4
+ *
5
+ * Both commands must read the same reference, or a pattern that looks precise in one report and
6
+ * harmful in the other is only a difference in which annotations were counted.
7
+ */
8
+ Object.defineProperty(exports, "__esModule", { value: true });
9
+ exports.loadReference = loadReference;
10
+ exports.segmentLookup = segmentLookup;
11
+ const types_1 = require("../types");
12
+ /**
13
+ * Latest annotation per (path, block) for the chosen model, grouped by message, so re-annotated
14
+ * messages do not count twice. An explicit prompt version pins the reference to labels made under
15
+ * one convention; without it, the newest label per block wins, whatever prompt produced it, and
16
+ * messages labelled only under an older convention are still included. The annotation guide's rule
17
+ * that runs made under different conventions must never be silently compared is the reason the
18
+ * option exists; use it whenever more than one prompt version is present.
19
+ */
20
+ function loadReference(store, model, promptVersion = '') {
21
+ const versionClause = promptVersion ? ' AND r.prompt_version = ?' : '';
22
+ const versionClauseInner = promptVersion ? ' AND r2.prompt_version = ?' : '';
23
+ const params = promptVersion ? [model, promptVersion, promptVersion] : [model];
24
+ const reference = store.query(`SELECT a.path, a.block_index, a.mime_part, a.label, a.char_count, m.client, m.lang, m.reply_style
25
+ FROM annotations a
26
+ JOIN annotation_runs r ON r.id = a.run_id
27
+ JOIN messages m ON m.path = a.path
28
+ WHERE r.model = ? AND a.source = 'llm'${versionClause}
29
+ AND a.id = (SELECT MAX(a2.id) FROM annotations a2
30
+ JOIN annotation_runs r2 ON r2.id = a2.run_id
31
+ WHERE a2.path = a.path AND a2.block_index = a.block_index AND r2.model = r.model${versionClauseInner})
32
+ ORDER BY a.path, a.block_index`, ...params);
33
+ const byPath = new Map();
34
+ for (const row of reference) {
35
+ const list = byPath.get(row.path);
36
+ if (list) {
37
+ list.push(row);
38
+ }
39
+ else {
40
+ byPath.set(row.path, [row]);
41
+ }
42
+ }
43
+ return byPath;
44
+ }
45
+ /**
46
+ * Map a stored (part, block index) to the output segment covering it. Segments merge adjacent
47
+ * blocks, so a block is matched by its index falling inside a segment's block range.
48
+ *
49
+ * Keyed per MIME part, not by block index alone. Block indices are section-local, so in a multipart
50
+ * message with several body sections two segments legitimately start at index 0, and a flat map
51
+ * silently kept only the last of them. Older annotations predate the part being recorded; those
52
+ * fall back to the first part the output used, which is exactly the previous behaviour for the
53
+ * single section messages they came from.
54
+ */
55
+ function segmentLookup(...segmentLists) {
56
+ const predictions = new Map();
57
+ for (const segment of segmentLists.flat()) {
58
+ const blockIndex = (0, types_1.blockIndexOf)(segment.source);
59
+ if (blockIndex === null) {
60
+ continue;
61
+ }
62
+ const part = (0, types_1.mimePartOf)(segment.source) ?? '';
63
+ let bucket = predictions.get(part);
64
+ if (!bucket) {
65
+ bucket = new Map();
66
+ predictions.set(part, bucket);
67
+ }
68
+ bucket.set(blockIndex, { label: segment.type, hidden: segment.visibility === 'hide' });
69
+ }
70
+ const ordered = new Map([...predictions].map(([part, bucket]) => [part, [...bucket.keys()].sort((a, b) => a - b)]));
71
+ // Segments start at a block index and run to the next one, so fill forward within the part.
72
+ return (part, index) => {
73
+ const key = part !== null && predictions.has(part) ? part : (predictions.keys().next().value ?? '');
74
+ const starts = ordered.get(key) ?? [];
75
+ let best = null;
76
+ for (const start of starts) {
77
+ if (start <= index) {
78
+ best = start;
79
+ }
80
+ else {
81
+ break;
82
+ }
83
+ }
84
+ return best === null ? null : (predictions.get(key)?.get(best) ?? null);
85
+ };
86
+ }
87
+ //# sourceMappingURL=reference.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reference.js","sourceRoot":"","sources":["../../src/eval/reference.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAyBH,sCA2BC;AAiBD,sCA+BC;AAjGD,oCAA8E;AAc9E;;;;;;;GAOG;AACH,SAAgB,aAAa,CAAC,KAAkB,EAAE,KAAa,EAAE,aAAa,GAAG,EAAE;IAC/E,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,MAAM,kBAAkB,GAAG,aAAa,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7E,MAAM,MAAM,GAAa,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACzF,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CACzB;;;;iDAIyC,aAAa;;;yGAG2C,kBAAkB;wCACnF,EAChC,GAAG,MAAM,CACZ,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAOD;;;;;;;;;GASG;AACH,SAAgB,aAAa,CAAC,GAAG,YAA6C;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2C,CAAC;IACvE,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACtB,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,IAAA,kBAAU,EAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACnB,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpH,4FAA4F;IAC5F,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnB,MAAM,GAAG,GAAG,IAAI,KAAK,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACpG,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,GAAkB,IAAI,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBACjB,IAAI,GAAG,KAAK,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,MAAM;YACV,CAAC;QACL,CAAC;QACD,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5E,CAAC,CAAC;AACN,CAAC"}
package/dist/index.d.ts CHANGED
@@ -18,7 +18,8 @@ export { buildHistoryDocument, matchBlocks, type HistoryDocument, type BlockMatc
18
18
  export { corpusThreadResolver, folderThreadResolver, type ThreadResolver } from './corpus/thread-index';
19
19
  export { analyze, resolveHistory, senderTokensFrom, type BodySection, type MessageContent, type AnalyzeOptions, type AnalyzeResult } from './analyze';
20
20
  export { planBody, reportableExclusions, type BodyPlan, type ExclusionReason, type PlannedSection, type SectionRole } from './mime/select';
21
- export { loadPatterns, PatternAssetError } from './detect/patterns';
21
+ export { loadPatterns, patternSource, PatternAssetError, type PatternSource } from './detect/patterns';
22
+ export { canonicalisePlainText } from './canonical/text';
22
23
  export interface ExtractOptions {
23
24
  parse?: ParseOptions;
24
25
  confidence?: Partial<ConfidenceConfig>;
@@ -39,6 +40,9 @@ export interface ExplainedResult extends ExtractionResult {
39
40
  primaryPart: string | null;
40
41
  blocks: {
41
42
  index: number;
43
+ /** MIME part and section-local index, the coordinates stored annotations use. */
44
+ mimePart: string;
45
+ sourceIndex: number;
42
46
  kind: string;
43
47
  label: string;
44
48
  confidence: number;
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,YAAY,EAA4C,MAAM,cAAc,CAAC;AAG3F,OAAO,EAA4B,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,MAAM,EAAgB,MAAM,SAAS,CAAC;AAE9F,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC5G,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AACtJ,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAC3I,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAWpE,MAAM,WAAW,cAAc;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC;;;OAGG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAC9C,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACrD,qCAAqC;IACrC,WAAW,CAAC,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,MAAM,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,EAAE,MAAM,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAC;gBAAC,QAAQ,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;YAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC;SACnB,EAAE,CAAC;KACP,CAAC;CACL;AA2CD,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,CA2G1G"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,KAAK,YAAY,EAA4C,MAAM,cAAc,CAAC;AAG3F,OAAO,EAA4B,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,MAAM,EAAgB,MAAM,SAAS,CAAC;AAE9F,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC5G,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AACtJ,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAC3I,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvG,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAWzD,MAAM,WAAW,cAAc;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvC;;;OAGG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAC9C,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACrD,qCAAqC;IACrC,WAAW,CAAC,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,MAAM,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,iFAAiF;YACjF,QAAQ,EAAE,MAAM,CAAC;YACjB,WAAW,EAAE,MAAM,CAAC;YACpB,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,UAAU,EAAE,MAAM,CAAC;YACnB,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAC;gBAAC,QAAQ,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;YAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC;SACnB,EAAE,CAAC;KACP,CAAC;CACL;AA4CD,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,CA4G1G"}