@postalsys/email-content 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +288 -0
  3. package/dist/.tsbuildinfo +1 -0
  4. package/dist/analyze.d.ts +91 -0
  5. package/dist/analyze.d.ts.map +1 -0
  6. package/dist/analyze.js +274 -0
  7. package/dist/analyze.js.map +1 -0
  8. package/dist/annotate/client.d.ts +64 -0
  9. package/dist/annotate/client.d.ts.map +1 -0
  10. package/dist/annotate/client.js +139 -0
  11. package/dist/annotate/client.js.map +1 -0
  12. package/dist/annotate/llm.d.ts +46 -0
  13. package/dist/annotate/llm.d.ts.map +1 -0
  14. package/dist/annotate/llm.js +107 -0
  15. package/dist/annotate/llm.js.map +1 -0
  16. package/dist/annotate/prompt.d.ts +38 -0
  17. package/dist/annotate/prompt.d.ts.map +1 -0
  18. package/dist/annotate/prompt.js +165 -0
  19. package/dist/annotate/prompt.js.map +1 -0
  20. package/dist/annotate/store.d.ts +43 -0
  21. package/dist/annotate/store.d.ts.map +1 -0
  22. package/dist/annotate/store.js +46 -0
  23. package/dist/annotate/store.js.map +1 -0
  24. package/dist/canonical/html.d.ts +33 -0
  25. package/dist/canonical/html.d.ts.map +1 -0
  26. package/dist/canonical/html.js +156 -0
  27. package/dist/canonical/html.js.map +1 -0
  28. package/dist/canonical/text.d.ts +75 -0
  29. package/dist/canonical/text.d.ts.map +1 -0
  30. package/dist/canonical/text.js +217 -0
  31. package/dist/canonical/text.js.map +1 -0
  32. package/dist/classify/confidence.d.ts +38 -0
  33. package/dist/classify/confidence.d.ts.map +1 -0
  34. package/dist/classify/confidence.js +59 -0
  35. package/dist/classify/confidence.js.map +1 -0
  36. package/dist/classify/reconcile.d.ts +52 -0
  37. package/dist/classify/reconcile.d.ts.map +1 -0
  38. package/dist/classify/reconcile.js +100 -0
  39. package/dist/classify/reconcile.js.map +1 -0
  40. package/dist/classify/rules.d.ts +51 -0
  41. package/dist/classify/rules.d.ts.map +1 -0
  42. package/dist/classify/rules.js +648 -0
  43. package/dist/classify/rules.js.map +1 -0
  44. package/dist/cli/annotate.d.ts +23 -0
  45. package/dist/cli/annotate.d.ts.map +1 -0
  46. package/dist/cli/annotate.js +328 -0
  47. package/dist/cli/annotate.js.map +1 -0
  48. package/dist/cli/args.d.ts +17 -0
  49. package/dist/cli/args.d.ts.map +1 -0
  50. package/dist/cli/args.js +72 -0
  51. package/dist/cli/args.js.map +1 -0
  52. package/dist/cli/benchmark.d.ts +13 -0
  53. package/dist/cli/benchmark.d.ts.map +1 -0
  54. package/dist/cli/benchmark.js +172 -0
  55. package/dist/cli/benchmark.js.map +1 -0
  56. package/dist/cli/corpus.d.ts +9 -0
  57. package/dist/cli/corpus.d.ts.map +1 -0
  58. package/dist/cli/corpus.js +152 -0
  59. package/dist/cli/corpus.js.map +1 -0
  60. package/dist/cli/disagreements.d.ts +14 -0
  61. package/dist/cli/disagreements.d.ts.map +1 -0
  62. package/dist/cli/disagreements.js +133 -0
  63. package/dist/cli/disagreements.js.map +1 -0
  64. package/dist/cli/evaluate.d.ts +15 -0
  65. package/dist/cli/evaluate.d.ts.map +1 -0
  66. package/dist/cli/evaluate.js +215 -0
  67. package/dist/cli/evaluate.js.map +1 -0
  68. package/dist/cli/extract.d.ts +10 -0
  69. package/dist/cli/extract.d.ts.map +1 -0
  70. package/dist/cli/extract.js +139 -0
  71. package/dist/cli/extract.js.map +1 -0
  72. package/dist/cli/main.d.ts +9 -0
  73. package/dist/cli/main.d.ts.map +1 -0
  74. package/dist/cli/main.js +147 -0
  75. package/dist/cli/main.js.map +1 -0
  76. package/dist/corpus/analyze.d.ts +66 -0
  77. package/dist/corpus/analyze.d.ts.map +1 -0
  78. package/dist/corpus/analyze.js +214 -0
  79. package/dist/corpus/analyze.js.map +1 -0
  80. package/dist/corpus/migrations/index.d.ts +14 -0
  81. package/dist/corpus/migrations/index.d.ts.map +1 -0
  82. package/dist/corpus/migrations/index.js +158 -0
  83. package/dist/corpus/migrations/index.js.map +1 -0
  84. package/dist/corpus/sample.d.ts +66 -0
  85. package/dist/corpus/sample.d.ts.map +1 -0
  86. package/dist/corpus/sample.js +98 -0
  87. package/dist/corpus/sample.js.map +1 -0
  88. package/dist/corpus/scanner.d.ts +79 -0
  89. package/dist/corpus/scanner.d.ts.map +1 -0
  90. package/dist/corpus/scanner.js +256 -0
  91. package/dist/corpus/scanner.js.map +1 -0
  92. package/dist/corpus/stats.d.ts +76 -0
  93. package/dist/corpus/stats.d.ts.map +1 -0
  94. package/dist/corpus/stats.js +240 -0
  95. package/dist/corpus/stats.js.map +1 -0
  96. package/dist/corpus/store.d.ts +67 -0
  97. package/dist/corpus/store.d.ts.map +1 -0
  98. package/dist/corpus/store.js +196 -0
  99. package/dist/corpus/store.js.map +1 -0
  100. package/dist/corpus/thread-index.d.ts +31 -0
  101. package/dist/corpus/thread-index.d.ts.map +1 -0
  102. package/dist/corpus/thread-index.js +182 -0
  103. package/dist/corpus/thread-index.js.map +1 -0
  104. package/dist/corpus/worker.d.ts +21 -0
  105. package/dist/corpus/worker.d.ts.map +1 -0
  106. package/dist/corpus/worker.js +74 -0
  107. package/dist/corpus/worker.js.map +1 -0
  108. package/dist/detect/client.d.ts +28 -0
  109. package/dist/detect/client.d.ts.map +1 -0
  110. package/dist/detect/client.js +121 -0
  111. package/dist/detect/client.js.map +1 -0
  112. package/dist/detect/disclaimer.d.ts +12 -0
  113. package/dist/detect/disclaimer.d.ts.map +1 -0
  114. package/dist/detect/disclaimer.js +28 -0
  115. package/dist/detect/disclaimer.js.map +1 -0
  116. package/dist/detect/forward-header.d.ts +10 -0
  117. package/dist/detect/forward-header.d.ts.map +1 -0
  118. package/dist/detect/forward-header.js +35 -0
  119. package/dist/detect/forward-header.js.map +1 -0
  120. package/dist/detect/patterns/disclaimers.json +113 -0
  121. package/dist/detect/patterns/forward-headers.json +40 -0
  122. package/dist/detect/patterns/html-clients.json +157 -0
  123. package/dist/detect/patterns/quoted-headers.json +41 -0
  124. package/dist/detect/patterns/reply-headers.json +139 -0
  125. package/dist/detect/patterns/signatures.json +58 -0
  126. package/dist/detect/patterns.d.ts +43 -0
  127. package/dist/detect/patterns.d.ts.map +1 -0
  128. package/dist/detect/patterns.js +125 -0
  129. package/dist/detect/patterns.js.map +1 -0
  130. package/dist/detect/reply-header.d.ts +44 -0
  131. package/dist/detect/reply-header.d.ts.map +1 -0
  132. package/dist/detect/reply-header.js +134 -0
  133. package/dist/detect/reply-header.js.map +1 -0
  134. package/dist/detect/signature.d.ts +28 -0
  135. package/dist/detect/signature.d.ts.map +1 -0
  136. package/dist/detect/signature.js +77 -0
  137. package/dist/detect/signature.js.map +1 -0
  138. package/dist/detect/traffic.d.ts +26 -0
  139. package/dist/detect/traffic.d.ts.map +1 -0
  140. package/dist/detect/traffic.js +98 -0
  141. package/dist/detect/traffic.js.map +1 -0
  142. package/dist/eval/metrics.d.ts +87 -0
  143. package/dist/eval/metrics.d.ts.map +1 -0
  144. package/dist/eval/metrics.js +185 -0
  145. package/dist/eval/metrics.js.map +1 -0
  146. package/dist/index.d.ts +54 -0
  147. package/dist/index.d.ts.map +1 -0
  148. package/dist/index.js +163 -0
  149. package/dist/index.js.map +1 -0
  150. package/dist/match/history.d.ts +62 -0
  151. package/dist/match/history.d.ts.map +1 -0
  152. package/dist/match/history.js +128 -0
  153. package/dist/match/history.js.map +1 -0
  154. package/dist/match/normalize.d.ts +33 -0
  155. package/dist/match/normalize.d.ts.map +1 -0
  156. package/dist/match/normalize.js +63 -0
  157. package/dist/match/normalize.js.map +1 -0
  158. package/dist/match/shingle.d.ts +44 -0
  159. package/dist/match/shingle.d.ts.map +1 -0
  160. package/dist/match/shingle.js +78 -0
  161. package/dist/match/shingle.js.map +1 -0
  162. package/dist/mime/charset.d.ts +33 -0
  163. package/dist/mime/charset.d.ts.map +1 -0
  164. package/dist/mime/charset.js +118 -0
  165. package/dist/mime/charset.js.map +1 -0
  166. package/dist/mime/headers.d.ts +32 -0
  167. package/dist/mime/headers.d.ts.map +1 -0
  168. package/dist/mime/headers.js +127 -0
  169. package/dist/mime/headers.js.map +1 -0
  170. package/dist/mime/model.d.ts +111 -0
  171. package/dist/mime/model.d.ts.map +1 -0
  172. package/dist/mime/model.js +17 -0
  173. package/dist/mime/model.js.map +1 -0
  174. package/dist/mime/parse.d.ts +27 -0
  175. package/dist/mime/parse.d.ts.map +1 -0
  176. package/dist/mime/parse.js +343 -0
  177. package/dist/mime/parse.js.map +1 -0
  178. package/dist/segment/features.d.ts +29 -0
  179. package/dist/segment/features.d.ts.map +1 -0
  180. package/dist/segment/features.js +102 -0
  181. package/dist/segment/features.js.map +1 -0
  182. package/dist/segment/html-blocks.d.ts +28 -0
  183. package/dist/segment/html-blocks.d.ts.map +1 -0
  184. package/dist/segment/html-blocks.js +596 -0
  185. package/dist/segment/html-blocks.js.map +1 -0
  186. package/dist/segment/text-blocks.d.ts +30 -0
  187. package/dist/segment/text-blocks.d.ts.map +1 -0
  188. package/dist/segment/text-blocks.js +204 -0
  189. package/dist/segment/text-blocks.js.map +1 -0
  190. package/dist/segment/types.d.ts +88 -0
  191. package/dist/segment/types.d.ts.map +1 -0
  192. package/dist/segment/types.js +11 -0
  193. package/dist/segment/types.js.map +1 -0
  194. package/dist/types.d.ts +85 -0
  195. package/dist/types.d.ts.map +1 -0
  196. package/dist/types.js +17 -0
  197. package/dist/types.js.map +1 -0
  198. package/dist/util/hash.d.ts +49 -0
  199. package/dist/util/hash.d.ts.map +1 -0
  200. package/dist/util/hash.js +0 -0
  201. package/dist/util/hash.js.map +1 -0
  202. package/dist/util/lang.d.ts +25 -0
  203. package/dist/util/lang.d.ts.map +1 -0
  204. package/dist/util/lang.js +135 -0
  205. package/dist/util/lang.js.map +1 -0
  206. package/dist/util/worker-pool.d.ts +51 -0
  207. package/dist/util/worker-pool.d.ts.map +1 -0
  208. package/dist/util/worker-pool.js +155 -0
  209. package/dist/util/worker-pool.js.map +1 -0
  210. package/package.json +74 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse.js","sourceRoot":"","sources":["../../src/mime/parse.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;;;AAqIH,gDA0BC;AAQD,oCAgMC;AArWD,kDAA8C;AAE9C,sDAA8B;AAE9B,uCAA0C;AAC1C,uCAAyF;AACzF,mCAAwH;AAExH,oGAAoG;AACpG,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;AAaxD,SAAS,QAAQ,CAAC,MAA4C;IAC1D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAqB;IAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACnC,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,cAAc,CAAC,OAAoB,EAAE,QAAgB,EAAE,QAAmB;IACrF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAEjE,IAAI,eAAuB,CAAC;IAC5B,IAAI,CAAC;QACD,eAAe,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,QAAQ,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,yBAAyB;YAC/B,QAAQ,EAAE,IAAI,CAAC,MAAM;YACrB,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,SAAS,KAAK,UAAU,CAAC,GAAG,CAAC,EAAE;SAC9D,CAAC,CAAC;QACH,eAAe,GAAG,GAAG,CAAC;IAC1B,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,eAAe,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACpC,eAAe,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACxD,SAAS,GAAG,IAAI,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,QAAQ,QAAQ,EAAE,CAAC,CAAC;IACjH,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,uBAAa,EAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,iBAAiB;YACvB,QAAQ,EAAE,IAAI,CAAC,MAAM;YACrB,MAAM,EAAE,YAAY,IAAI,CAAC,OAAO,IAAI,MAAM,UAAU,OAAO,CAAC,WAAW,EAAE;SAC5E,CAAC,CAAC;IACP,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,kCAAkC;YACxC,QAAQ,EAAE,IAAI,CAAC,MAAM;YACrB,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,yBAAyB;SAC/D,CAAC,CAAC;IACP,CAAC;IAED,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IACxB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;QACnD,IAAI,CAAC;YACD,IAAI,GAAG,iBAAO,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACL,8FAA8F;QAClG,CAAC;IACL,CAAC;IAED,OAAO;QACH,IAAI;QACJ,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,eAAe,EAAE,IAAI,CAAC,OAAO;QAC7B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,SAAS;KACZ,CAAC;AACN,CAAC;AAED,4FAA4F;AAC5F,SAAS,UAAU,CAAC,IAAkB,EAAE,GAAW;IAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC;YACZ,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,UAAU,CAAC,GAAY;IAC5B,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACrC,CAAC;AAED,6FAA6F;AAC7F,SAAgB,kBAAkB,CAAC,KAA0B,EAAE,SAAwC,EAAE,MAAc;IACnH,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,KAAa,EAAU,EAAE;QACjD,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;YACb,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChF,OAAO,KAAK,IAAI,CAAC,SAAS,IAAI,QAAQ,GAAG,CAAC;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtC,OAAO,UAAU,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACvF,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;YACpC,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YACnC,OAAO,GAAG,CAAC;QACf,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1D,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,UAAwB,EAAE;IACtE,MAAM,IAAI,GAAG,EAAE,GAAG,6BAAqB,EAAE,GAAG,OAAO,EAAE,CAAC;IACtD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEjD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,MAAM,CAAC;IACpB,IAAI,cAAc,GAAoC,EAAE,CAAC;IACzD,IAAI,aAAa,GAAG,KAAK,CAAC;IAE1B,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;QAC9B,MAAM,QAAQ,GAAG,IAAI,oBAAQ,CAAC;YAC1B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,aAAa,EAAE,IAAI,CAAC,QAAQ;YAC5B,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAC;QAEH,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,MAAM,GAAG,GAAG,EAAE;YAChB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YACxB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAqB,CAAC;gBACnC,IAAI,WAAmB,CAAC;gBACxB,IAAI,CAAC;oBACD,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACrE,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC;gBAC3B,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC;gBAE7B,IAAI,YAAY,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACvC,IAAI,CAAC,aAAa,EAAE,CAAC;wBACjB,aAAa,GAAG,IAAI,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,iBAAiB,IAAI,CAAC,QAAQ,QAAQ,EAAE,CAAC,CAAC;oBAClG,CAAC;oBACD,OAAO;gBACX,CAAC;gBAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9D,MAAM,OAAO,GAAG,IAAA,yBAAe,EAAC,UAAU,CAAC,CAAC;gBAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,cAAc,CAAC,CAAC;gBAC9E,MAAM,QAAQ,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;gBAE/F,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAE3E,MAAM,IAAI,GAAa;oBACnB,MAAM;oBACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACjD,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,YAAY;oBACrF,iBAAiB,EAAE,QAAQ;oBAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;oBACnD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;oBACpE,WAAW,EAAE,oBAAoB,CAAC,IAAI,CAAC,WAAW,CAAC;oBACnD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;oBACtD,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;oBACzD,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC5B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC;oBACvC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC5B,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;oBAC1B,QAAQ;oBACR,QAAQ,EAAE,EAAE;oBACZ,WAAW,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE;oBAChD,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,CAAC;oBACd,OAAO,EAAE,OAAO;oBAChB,OAAO,EAAE,IAAI;iBAChB,CAAC;gBAEF,MAAM,OAAO,GAAgB;oBACzB,IAAI;oBACJ,IAAI;oBACJ,UAAU,EAAE,EAAE;oBACd,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;oBAC5C,aAAa,EAAE,KAAK;iBACvB,CAAC;gBAEF,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,MAAM,GAAG,MAAM,CAAC;oBAChB,cAAc,GAAG,UAAU,CAAC;gBAChC,CAAC;gBAED,iFAAiF;gBACjF,wCAAwC;gBACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC/B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3B,IAAI,QAAQ,EAAE,CAAC;wBACX,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACvC,IAAI,MAAM,EAAE,CAAC;4BACT,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACtC,CAAC;oBACL,CAAC;gBACL,CAAC;gBACD,OAAO;YACX,CAAC;YAED,MAAM,KAAK,GAAW,KAAK,CAAC,KAAK,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YAEvB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACvC,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,OAAO;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC7D,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;YAC/B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;YACzG,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;YACvC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACnB,qFAAqF;gBACrF,2DAA2D;gBAC3D,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;oBAC9C,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC7B,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;gBACpC,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,wFAAwF;QACxF,6FAA6F;QAC7F,4DAA4D;QAC5D,MAAM,SAAS,GAAG,QAAwF,CAAC;QAC3G,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAa,EAAE,EAAE;YACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,MAAM,EAAE,CAAC;QACb,CAAC,CAAC,CAAC;QACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC5B,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC1F,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG;gBACnB,IAAI,EAAE,EAAE;gBACR,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO;gBAC5C,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO;gBACrC,gBAAgB,EAAE,CAAC;gBACnB,SAAS,EAAE,KAAK;aACnB,CAAC;QACN,CAAC;QACD,yEAAyE;QACzE,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAA,yBAAe,EAAC,cAAc,CAAC,CAAC;IAEpD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO;QACH,KAAK;QACL,SAAS,EAAE,IAAI;QACf,MAAM;QACN,OAAO,EAAE,IAAA,+BAAqB,EAAC,WAAW,CAAC;QAC3C,WAAW;QACX,QAAQ;QACR,IAAI,EAAE,GAAG,CAAC,MAAM;QAChB,SAAS,EAAE,kBAAkB,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;KACrD,CAAC;AACN,CAAC"}
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Feature extraction for blocks.
3
+ *
4
+ * Every feature is a plain number, boolean or string list, and every one is dumped by
5
+ * `extract --explain`. A feature that cannot be shown to a human cannot be debugged when it causes
6
+ * content to disappear, and that is the failure mode this project cares most about.
7
+ */
8
+ import { type BlockFeatures } from './types';
9
+ export interface FeatureContext {
10
+ /** Zero based block index. */
11
+ index: number;
12
+ /** Total blocks in this representation. */
13
+ total: number;
14
+ /** Character offset of the block start. */
15
+ start: number;
16
+ /** Total characters in the representation. */
17
+ totalChars: number;
18
+ quoteDepth: number;
19
+ blockquoteDepth?: number;
20
+ elementCount?: number;
21
+ tags?: string[];
22
+ classes?: string[];
23
+ hasTopBorder?: boolean;
24
+ linkCount?: number;
25
+ imageCount?: number;
26
+ }
27
+ /** Compute the feature vector for a block's de-prefixed content. */
28
+ export declare function computeFeatures(content: string, context: FeatureContext): BlockFeatures;
29
+ //# sourceMappingURL=features.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"features.d.ts","sourceRoot":"","sources":["../../src/segment/features.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,SAAS,CAAC;AAwC7C,MAAM,WAAW,cAAc;IAC3B,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAiBD,oEAAoE;AACpE,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,aAAa,CA6CvF"}
@@ -0,0 +1,102 @@
1
+ "use strict";
2
+ /**
3
+ * Feature extraction for blocks.
4
+ *
5
+ * Every feature is a plain number, boolean or string list, and every one is dumped by
6
+ * `extract --explain`. A feature that cannot be shown to a human cannot be debugged when it causes
7
+ * content to disappear, and that is the failure mode this project cares most about.
8
+ */
9
+ Object.defineProperty(exports, "__esModule", { value: true });
10
+ exports.computeFeatures = computeFeatures;
11
+ const patterns_1 = require("../detect/patterns");
12
+ /**
13
+ * Bounded on purpose. The unbounded form, [\w.+-]+@..., backtracks quadratically over a long run of
14
+ * word characters that contains no @, which is exactly what a pasted log or a base64 blob looks
15
+ * like. It took 21 seconds on a 200 KB block. The bounds are generous relative to real addresses.
16
+ */
17
+ const EMAIL = /[\w.+-]{1,64}@[\w-]{1,63}\.[\w.-]{2,64}/;
18
+ const URL = /\b(?:https?:\/\/|www\.)\S+/i;
19
+ const PHONE = /(?:\+\d[\d\s().-]{7,}|\b\d{3}[\s.-]\d{3}[\s.-]\d{3,4}\b)/;
20
+ /**
21
+ * Date-like patterns.
22
+ *
23
+ * Attribution lines nearly always carry a date, so this feature carries real weight. It covers ISO,
24
+ * European and US numeric forms plus English month names, and deliberately stops there: a fully
25
+ * multilingual date matcher would be large, and the attribution patterns already carry the language
26
+ * specific work.
27
+ */
28
+ const DATE = /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}[./]\d{1,2}[./]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}\b|\b\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\b|\b\d{1,2}:\d{2}\b)/i;
29
+ /** Characters inspected when computing character class ratios. */
30
+ const CHARACTER_SAMPLE = 8192;
31
+ const UPPER_RE = /\p{Lu}/gu;
32
+ const LETTER_RE = /\p{L}/gu;
33
+ const PUNCTUATION_RE = /[\p{P}\p{S}]/gu;
34
+ /** Count regex matches without materialising the match array. */
35
+ function countMatches(text, regex) {
36
+ regex.lastIndex = 0;
37
+ let count = 0;
38
+ while (regex.exec(text) !== null) {
39
+ count++;
40
+ }
41
+ regex.lastIndex = 0;
42
+ return count;
43
+ }
44
+ function countHeaderFieldLines(lines) {
45
+ const patterns = (0, patterns_1.loadPatterns)();
46
+ let count = 0;
47
+ for (const line of lines) {
48
+ const trimmed = line.trim();
49
+ if (!trimmed || trimmed.length > 300) {
50
+ continue;
51
+ }
52
+ if (patterns.quotedHeaderFields.some(pattern => pattern.regex.test(trimmed))) {
53
+ count++;
54
+ }
55
+ }
56
+ return count;
57
+ }
58
+ /** Compute the feature vector for a block's de-prefixed content. */
59
+ function computeFeatures(content, context) {
60
+ const lines = content.split('\n');
61
+ const nonBlank = lines.filter(line => line.trim().length > 0);
62
+ const charCount = content.length;
63
+ // Character class ratios are computed over a bounded prefix, in bulk.
64
+ //
65
+ // These are statistical features, so a sample is as informative as the whole block, and the
66
+ // bound matters: the previous per character version tested three Unicode property regexes for
67
+ // every character, which took 16 seconds on a 200 KB plain text body. That is a realistic body
68
+ // size for a pasted log, and a library that hangs for 16 seconds on one is unusable.
69
+ const sample = content.length > CHARACTER_SAMPLE ? content.slice(0, CHARACTER_SAMPLE) : content;
70
+ const upper = countMatches(sample, UPPER_RE);
71
+ const letters = countMatches(sample, LETTER_RE);
72
+ const punctuation = countMatches(sample, PUNCTUATION_RE);
73
+ const shortLines = nonBlank.filter(line => line.trim().length < 40).length;
74
+ const words = content.split(/\s+/).filter(Boolean);
75
+ return {
76
+ position: context.totalChars > 0 ? context.start / context.totalChars : 0,
77
+ indexFromEnd: Math.max(0, context.total - 1 - context.index),
78
+ charCount,
79
+ lineCount: lines.length,
80
+ meanLineLength: nonBlank.length ? nonBlank.reduce((sum, line) => sum + line.length, 0) / nonBlank.length : 0,
81
+ wordCount: words.length,
82
+ quoteDepth: context.quoteDepth,
83
+ blockquoteDepth: context.blockquoteDepth ?? 0,
84
+ elementCount: context.elementCount ?? 0,
85
+ linkCount: context.linkCount ?? 0,
86
+ imageCount: context.imageCount ?? 0,
87
+ hasQuestionMark: content.includes('?'),
88
+ hasDate: DATE.test(content),
89
+ hasEmailAddress: EMAIL.test(content),
90
+ hasUrl: URL.test(content),
91
+ hasPhoneNumber: PHONE.test(content),
92
+ headerFieldLines: countHeaderFieldLines(lines),
93
+ shortLineRatio: nonBlank.length ? shortLines / nonBlank.length : 0,
94
+ upperCaseRatio: letters ? upper / letters : 0,
95
+ punctuationRatio: charCount ? punctuation / charCount : 0,
96
+ isAllBlank: nonBlank.length === 0,
97
+ tags: context.tags ?? [],
98
+ classes: context.classes ?? [],
99
+ hasTopBorder: context.hasTopBorder ?? false
100
+ };
101
+ }
102
+ //# sourceMappingURL=features.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"features.js","sourceRoot":"","sources":["../../src/segment/features.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;AA8EH,0CA6CC;AAzHD,iDAAkD;AAGlD;;;;GAIG;AACH,MAAM,KAAK,GAAG,yCAAyC,CAAC;AACxD,MAAM,GAAG,GAAG,6BAA6B,CAAC;AAC1C,MAAM,KAAK,GAAG,0DAA0D,CAAC;AAEzE;;;;;;;GAOG;AACH,MAAM,IAAI,GACN,iOAAiO,CAAC;AAEtO,kEAAkE;AAClE,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,QAAQ,GAAG,UAAU,CAAC;AAC5B,MAAM,SAAS,GAAG,SAAS,CAAC;AAC5B,MAAM,cAAc,GAAG,gBAAgB,CAAC;AAExC,iEAAiE;AACjE,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC7C,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,KAAK,EAAE,CAAC;IACZ,CAAC;IACD,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;IACpB,OAAO,KAAK,CAAC;AACjB,CAAC;AAqBD,SAAS,qBAAqB,CAAC,KAAwB;IACnD,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACnC,SAAS;QACb,CAAC;QACD,IAAI,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAC3E,KAAK,EAAE,CAAC;QACZ,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,oEAAoE;AACpE,SAAgB,eAAe,CAAC,OAAe,EAAE,OAAuB;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjC,sEAAsE;IACtE,EAAE;IACF,4FAA4F;IAC5F,8FAA8F;IAC9F,+FAA+F;IAC/F,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAChG,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAEzD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnD,OAAO;QACH,QAAQ,EAAE,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACzE,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5D,SAAS;QACT,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5G,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,CAAC;QAC7C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC;QACvC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC;QACjC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;QACnC,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;QACtC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAC3B,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;QACpC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;QACzB,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;QACnC,gBAAgB,EAAE,qBAAqB,CAAC,KAAK,CAAC;QAC9C,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAClE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,gBAAgB,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACzD,UAAU,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;QACjC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;QAC9B,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK;KAC9C,CAAC;AACN,CAAC"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Stage 3: HTML block segmentation.
3
+ *
4
+ * The DOM is walked with node-html-parser, which exposes a source `range` on every node. That is
5
+ * the reason it is used here rather than a faster tag stripper: a block that cannot point back at
6
+ * the exact source offsets it came from cannot be traced, and traceability is a requirement.
7
+ *
8
+ * Segmentation emits one block per run of inline content between block level elements. A container
9
+ * such as a Gmail quote wrapper does not become a block itself; instead every block inside it
10
+ * records its ancestry. That is deliberate, because interleaved replies put new content inside and
11
+ * between quote containers, and a container-level decision cannot express that.
12
+ *
13
+ * Blocks tile the source: each block's range is extended backwards to the end of the previous
14
+ * block, and the last block runs to the end of the document, so concatenating block texts
15
+ * reproduces the HTML exactly.
16
+ */
17
+ import { type Block } from './types';
18
+ export interface HtmlSegmentation {
19
+ /** The source HTML, unchanged. All block offsets refer to this string. */
20
+ source: string;
21
+ blocks: Block[];
22
+ /** True when parsing failed and a degraded single block was produced. */
23
+ degraded: boolean;
24
+ }
25
+ /** True when markup is unbalanced enough that parsing it is not worth the time. */
26
+ export declare function isPathologicalHtml(html: string): boolean;
27
+ export declare function segmentHtmlBlocks(html: string, mimePart: string): HtmlSegmentation;
28
+ //# sourceMappingURL=html-blocks.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"html-blocks.d.ts","sourceRoot":"","sources":["../../src/segment/html-blocks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAUH,OAAO,EAAE,KAAK,KAAK,EAAiC,MAAM,SAAS,CAAC;AAmWpE,MAAM,WAAW,gBAAgB;IAC7B,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAC;CACrB;AA+BD,mFAAmF;AACnF,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,CAyOlF"}