@dvaji/cheerox 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1091 -0
  3. package/Cargo.toml +47 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7581 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BKkfJEAr.mjs +1648 -0
  18. package/dist/cheerio-compat-BKkfJEAr.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DcnYK9OB.cjs +1719 -0
  23. package/dist/cheerio-compat-DcnYK9OB.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4120 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +35 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/native/vendor/scraper-0.24.0/Cargo.toml +52 -0
  167. package/native/vendor/scraper-0.24.0/LICENSE +15 -0
  168. package/native/vendor/scraper-0.24.0/src/element_ref/element.rs +212 -0
  169. package/native/vendor/scraper-0.24.0/src/element_ref/mod.rs +236 -0
  170. package/native/vendor/scraper-0.24.0/src/element_ref/serializable.rs +15 -0
  171. package/native/vendor/scraper-0.24.0/src/error/utils.rs +91 -0
  172. package/native/vendor/scraper-0.24.0/src/error.rs +118 -0
  173. package/native/vendor/scraper-0.24.0/src/html/mod.rs +270 -0
  174. package/native/vendor/scraper-0.24.0/src/html/serializable.rs +27 -0
  175. package/native/vendor/scraper-0.24.0/src/html/tree_sink.rs +285 -0
  176. package/native/vendor/scraper-0.24.0/src/lib.rs +180 -0
  177. package/native/vendor/scraper-0.24.0/src/node/serializable.rs +52 -0
  178. package/native/vendor/scraper-0.24.0/src/node.rs +465 -0
  179. package/native/vendor/scraper-0.24.0/src/selectable.rs +70 -0
  180. package/native/vendor/scraper-0.24.0/src/selector.rs +290 -0
  181. package/package.json +167 -4
  182. package/scripts/check-native.mjs +15 -0
  183. package/scripts/check-release.mjs +234 -0
  184. package/scripts/fix-declarations.mjs +39 -0
  185. package/scripts/generate-notices.mjs +194 -0
  186. package/scripts/notices/README.md +13 -0
  187. package/scripts/notices/cheerio-MIT.txt +21 -0
  188. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  189. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  190. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  191. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  192. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  193. package/scripts/notices/napi-rs-MIT.txt +43 -0
  194. package/scripts/notices/neon-MIT.txt +19 -0
  195. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  196. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  197. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  198. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  199. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  200. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  201. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  202. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  203. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  204. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  205. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  206. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  207. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  208. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  209. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  210. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  211. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  212. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  213. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  214. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  215. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  216. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  217. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  218. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  219. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  220. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  221. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  222. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  223. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  224. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  225. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  226. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  227. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  228. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  229. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  230. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  231. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  232. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  233. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  234. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  235. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  236. package/scripts/notices/runtime.json +1632 -0
  237. package/scripts/notices/sources.json +561 -0
  238. package/scripts/notices/xray-MIT.txt +21 -0
  239. package/src/attribute-object.ts +71 -0
  240. package/src/attributes.ts +308 -0
  241. package/src/cheerio-compat/css.ts +58 -0
  242. package/src/cheerio-compat/render.ts +106 -0
  243. package/src/cheerio-compat/selectors.ts +51 -0
  244. package/src/cheerio-compat/types.ts +52 -0
  245. package/src/cheerio-compat.ts +2055 -0
  246. package/src/codec.ts +96 -0
  247. package/src/dom.ts +87 -0
  248. package/src/extract-plan.ts +97 -0
  249. package/src/extract.ts +322 -0
  250. package/src/hooks.ts +369 -0
  251. package/src/index.ts +48 -0
  252. package/src/io.ts +491 -0
  253. package/src/match.ts +207 -0
  254. package/src/modern.ts +259 -0
  255. package/src/native.ts +149 -0
  256. package/src/nodes.ts +831 -0
  257. package/src/parse.ts +53 -0
  258. package/src/parser-types.ts +157 -0
  259. package/src/router.ts +1129 -0
  260. package/src/selection.ts +682 -0
  261. package/src/slim.ts +12 -0
  262. package/src/snapshot.ts +291 -0
  263. package/src/transport.ts +113 -0
  264. package/src/types.ts +186 -0
  265. package/src/utils.ts +29 -0
  266. package/src/workers.ts +21 -0
  267. package/tsconfig.json +15 -0
  268. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,277 @@
1
+ use crate::utils::trace_log;
2
+ use crate::{Emitter, Reader, State, Tokenizer};
3
+
4
+ #[cfg(feature = "afl-ijon")]
5
+ use afl::{ijon_hashint, ijon_state};
6
+
7
+ #[derive(Debug)]
8
+ pub(crate) struct MachineState<R: Reader, E: Emitter> {
9
+ #[allow(clippy::type_complexity)]
10
+ pub function: fn(&mut Tokenizer<R, E>) -> Result<ControlToken<R, E>, R::Error>,
11
+ #[cfg(debug_assertions)]
12
+ pub debug_name: &'static str,
13
+ }
14
+
15
+ impl<R: Reader, E: Emitter> Copy for MachineState<R, E> {}
16
+ impl<R: Reader, E: Emitter> Clone for MachineState<R, E> {
17
+ fn clone(&self) -> Self {
18
+ *self
19
+ }
20
+ }
21
+
22
+ pub(crate) enum ControlToken<R: Reader, E: Emitter> {
23
+ Eof,
24
+ Continue,
25
+ SwitchTo(MachineState<R, E>),
26
+ }
27
+
28
+ impl<R: Reader, E: Emitter> ControlToken<R, E> {
29
+ #[inline(always)]
30
+ pub(crate) fn inline_next_state(
31
+ self,
32
+ tokenizer: &mut Tokenizer<R, E>,
33
+ ) -> Result<Self, R::Error> {
34
+ match self {
35
+ ControlToken::SwitchTo(state) => {
36
+ tokenizer.machine_helper.switch_to(state);
37
+ (state.function)(tokenizer)
38
+ }
39
+ _ => {
40
+ #[cfg(debug_assertions)]
41
+ panic!("use of inline_next_state is invalid in this context as no state switch is happening");
42
+
43
+ #[cfg(not(debug_assertions))]
44
+ Ok(self)
45
+ }
46
+ }
47
+ }
48
+ }
49
+
50
+ #[allow(clippy::from_over_into)]
51
+ impl<R: Reader, E: Emitter> Into<MachineState<R, E>> for State {
52
+ fn into(self) -> MachineState<R, E> {
53
+ // TODO: instead of this conversion, can we rig the enums to be of same layout?
54
+ match self {
55
+ State::Data => state_ref!(Data),
56
+ State::PlainText => state_ref!(PlainText),
57
+ State::RcData => state_ref!(RcData),
58
+ State::RawText => state_ref!(RawText),
59
+ State::ScriptData => state_ref!(ScriptData),
60
+ State::CdataSection => state_ref!(CdataSection),
61
+ }
62
+ }
63
+ }
64
+
65
+ #[derive(Debug)]
66
+ pub(crate) struct MachineHelper<R: Reader, E: Emitter> {
67
+ // XXX: allocation that cannot be controlled/reused by the user
68
+ pub(crate) temporary_buffer: Vec<u8>,
69
+ pub(crate) character_reference_code: u32,
70
+ pub(crate) state: MachineState<R, E>,
71
+ return_state: Option<(MachineState<R, E>, bool)>,
72
+ }
73
+
74
+ impl<R: Reader, E: Emitter> Default for MachineHelper<R, E> {
75
+ fn default() -> Self {
76
+ MachineHelper {
77
+ temporary_buffer: Vec::new(),
78
+ character_reference_code: 0,
79
+ state: state_ref!(Data),
80
+ return_state: None,
81
+ }
82
+ }
83
+ }
84
+
85
+ impl<R: Reader, E: Emitter> MachineHelper<R, E> {
86
+ pub(crate) fn is_consumed_as_part_of_an_attribute(&self) -> bool {
87
+ match self.return_state {
88
+ Some((_state, is_attribute)) => is_attribute,
89
+ None => false,
90
+ }
91
+ }
92
+
93
+ pub(crate) fn flush_code_points_consumed_as_character_reference(&mut self, emitter: &mut E) {
94
+ emitter.emit_char_ref(
95
+ &self.temporary_buffer,
96
+ self.is_consumed_as_part_of_an_attribute(),
97
+ );
98
+ self.temporary_buffer.clear();
99
+ }
100
+
101
+ pub(crate) fn flush_buffer_characters(&mut self, emitter: &mut E) {
102
+ emitter.emit_string(&self.temporary_buffer);
103
+ self.temporary_buffer.clear();
104
+ }
105
+
106
+ pub(crate) fn enter_state(&mut self, state: MachineState<R, E>, is_attribute: bool) {
107
+ debug_assert!(self.return_state.is_none());
108
+ let return_state = (self.state, is_attribute);
109
+ self.return_state = Some(return_state);
110
+ self.switch_to(state);
111
+ #[cfg(feature = "afl-ijon")]
112
+ ijon_state!(ijon_hashint(
113
+ ijon_hashint(state.function as u32, return_state.0.function as u32),
114
+ is_attribute as u32
115
+ ));
116
+ }
117
+
118
+ pub(crate) fn pop_return_state(&mut self) -> MachineState<R, E> {
119
+ self.return_state.take().unwrap().0
120
+ }
121
+
122
+ pub(crate) fn exit_state(&mut self) {
123
+ let state = self.pop_return_state();
124
+ self.switch_to(state);
125
+ }
126
+
127
+ pub(crate) fn switch_to(&mut self, state: MachineState<R, E>) {
128
+ trace_log!(
129
+ "switch_to: {} -> {}",
130
+ self.state.debug_name,
131
+ state.debug_name
132
+ );
133
+ self.state = state;
134
+ #[cfg(feature = "afl-ijon")]
135
+ ijon_state!(state.function as u32);
136
+ }
137
+ }
138
+
139
+ macro_rules! state_ref {
140
+ ($state:ident) => {{
141
+ crate::machine_helper::MachineState {
142
+ function: crate::machine::states::$state::run,
143
+ #[cfg(debug_assertions)]
144
+ debug_name: stringify!($state),
145
+ }
146
+ }};
147
+ }
148
+
149
+ pub(crate) use state_ref;
150
+
151
+ macro_rules! mutate_character_reference {
152
+ ($slf:expr, * $mul:literal + $x:ident - $sub:literal) => {
153
+ match $slf
154
+ .machine_helper
155
+ .character_reference_code
156
+ .checked_mul($mul)
157
+ .and_then(|cr| cr.checked_add($x as u32 - $sub))
158
+ {
159
+ Some(cr) => $slf.machine_helper.character_reference_code = cr,
160
+ None => {
161
+ // provoke err
162
+ $slf.machine_helper.character_reference_code = 0x110000;
163
+ }
164
+ };
165
+ };
166
+ }
167
+
168
+ pub(crate) use mutate_character_reference;
169
+
170
+ macro_rules! emit_current_tag_and_switch_to {
171
+ ($slf:expr, $state:ident) => {{
172
+ let state = $slf.emitter.emit_current_tag().map(Into::into);
173
+ if state.is_some() {
174
+ crate::utils::trace_log!("emitter asked for state switch:");
175
+ }
176
+ $slf.machine_helper
177
+ .switch_to(state.unwrap_or($crate::machine_helper::state_ref!($state)));
178
+ Ok(ControlToken::Continue)
179
+ }};
180
+ }
181
+
182
+ pub(crate) use emit_current_tag_and_switch_to;
183
+
184
+ macro_rules! switch_to {
185
+ ($slf:expr, $state:ident) => {{
186
+ let new_state = $crate::machine_helper::state_ref!($state);
187
+ Ok(ControlToken::SwitchTo(new_state))
188
+ }};
189
+ }
190
+
191
+ pub(crate) use switch_to;
192
+
193
+ macro_rules! enter_state {
194
+ ($slf:expr, $state:ident, $is_attribute:expr) => {{
195
+ $slf.machine_helper
196
+ .enter_state($crate::machine_helper::state_ref!($state), $is_attribute);
197
+ Ok(ControlToken::Continue)
198
+ }};
199
+ }
200
+
201
+ pub(crate) use enter_state;
202
+
203
+ macro_rules! exit_state {
204
+ ($slf:expr) => {{
205
+ $slf.machine_helper.exit_state();
206
+ Ok(ControlToken::Continue)
207
+ }};
208
+ }
209
+
210
+ pub(crate) use exit_state;
211
+
212
+ macro_rules! reconsume_in {
213
+ ($slf:expr, $c:expr, $state:ident) => {{
214
+ let new_state = $crate::machine_helper::state_ref!($state);
215
+ let c = $c;
216
+ $slf.unread_byte(c);
217
+ Ok(ControlToken::SwitchTo(new_state))
218
+ }};
219
+ }
220
+
221
+ pub(crate) use reconsume_in;
222
+
223
+ macro_rules! reconsume_in_return_state {
224
+ ($slf:expr, $c:expr) => {{
225
+ let new_state = $slf.machine_helper.pop_return_state();
226
+ let c = $c;
227
+ $slf.unread_byte(c);
228
+ Ok(ControlToken::SwitchTo(new_state))
229
+ }};
230
+ }
231
+
232
+ pub(crate) use reconsume_in_return_state;
233
+
234
+ macro_rules! cont {
235
+ () => {{
236
+ continue;
237
+ }};
238
+ }
239
+
240
+ pub(crate) use cont;
241
+
242
+ macro_rules! eof {
243
+ () => {{
244
+ Ok(ControlToken::Eof)
245
+ }};
246
+ }
247
+
248
+ pub(crate) use eof;
249
+
250
+ macro_rules! read_byte {
251
+ ($slf:expr) => {
252
+ $slf.read_byte()
253
+ };
254
+ }
255
+
256
+ pub(crate) use read_byte;
257
+
258
+ /// Produce error for current character. The error will be emitted once the character's bytes
259
+ /// have been fully consumed (and after any errors originating from pre-processing the input
260
+ /// stream bytes)
261
+ macro_rules! error {
262
+ ($slf:expr, $e:expr) => {
263
+ $slf.validator.set_character_error(&mut $slf.emitter, $e);
264
+ };
265
+ }
266
+
267
+ pub(crate) use error;
268
+
269
+ /// Produce error for a previous character, emit immediately.
270
+ macro_rules! error_immediate {
271
+ ($slf:expr, $e:expr) => {
272
+ error!($slf, $e);
273
+ $slf.validator.flush_character_error(&mut $slf.emitter);
274
+ };
275
+ }
276
+
277
+ pub(crate) use error_immediate;
@@ -0,0 +1,244 @@
1
+ use crate::char_validator::CharValidator;
2
+ use crate::Emitter;
3
+ use crate::Reader;
4
+
5
+ #[derive(Debug)]
6
+ pub(crate) struct ReadHelper<R: Reader> {
7
+ pub(crate) reader: R,
8
+ last_character_was_cr: bool,
9
+ #[allow(clippy::option_option)]
10
+ to_reconsume: Option<Option<u8>>,
11
+ }
12
+
13
+ impl<R: Reader> ReadHelper<R> {
14
+ pub(crate) fn new(reader: R) -> Self {
15
+ ReadHelper {
16
+ reader,
17
+ last_character_was_cr: false,
18
+ to_reconsume: None,
19
+ }
20
+ }
21
+
22
+ #[inline(always)]
23
+ pub(crate) fn read_byte<E: Emitter>(
24
+ &mut self,
25
+ char_validator: &mut CharValidator,
26
+ emitter: &mut E,
27
+ ) -> Result<Option<u8>, R::Error> {
28
+ if let Some(c) = self.to_reconsume.take() {
29
+ return Ok(c);
30
+ }
31
+
32
+ let mut c = self.reader.read_byte();
33
+ if self.last_character_was_cr && matches!(c, Ok(Some(b'\n'))) {
34
+ // TODO: we are scattering move_position in the read
35
+ // path across many places: read helper, tokenizer.rs,
36
+ // ... consolidate!
37
+ emitter.move_position(1);
38
+ c = self.reader.read_byte();
39
+ }
40
+
41
+ if matches!(c, Ok(Some(b'\r'))) {
42
+ self.last_character_was_cr = true;
43
+ c = Ok(Some(b'\n'));
44
+ } else {
45
+ self.last_character_was_cr = false;
46
+ }
47
+
48
+ if let Ok(Some(x)) = c {
49
+ char_validator.validate_byte(emitter, x);
50
+ }
51
+
52
+ c
53
+ }
54
+
55
+ #[inline(always)]
56
+ pub(crate) fn try_read_string(
57
+ &mut self,
58
+ char_validator: &mut CharValidator,
59
+ mut s: &str,
60
+ case_sensitive: bool,
61
+ ) -> Result<bool, R::Error> {
62
+ debug_assert!(!s.is_empty());
63
+ debug_assert!(!s.contains('\r'));
64
+
65
+ let to_reconsume_bak = self.to_reconsume;
66
+ let mut bytes = s.as_bytes().iter();
67
+ if let Some(c) = self.to_reconsume.take() {
68
+ match (c, bytes.next()) {
69
+ (Some(x), Some(&x2))
70
+ if x == x2 || (!case_sensitive && x.eq_ignore_ascii_case(&x2)) =>
71
+ {
72
+ s = &s[1..];
73
+ }
74
+ _ => {
75
+ self.to_reconsume = to_reconsume_bak;
76
+ return Ok(false);
77
+ }
78
+ }
79
+ }
80
+
81
+ if s.is_empty() || self.reader.try_read_string(s.as_bytes(), case_sensitive)? {
82
+ self.last_character_was_cr = false;
83
+ char_validator.reset();
84
+ Ok(true)
85
+ } else {
86
+ self.to_reconsume = to_reconsume_bak;
87
+ Ok(false)
88
+ }
89
+ }
90
+
91
+ #[inline(always)]
92
+ pub(crate) fn read_until<'b, E: Emitter>(
93
+ &'b mut self,
94
+ needle: &[u8],
95
+ char_validator: &mut CharValidator,
96
+ emitter: &mut E,
97
+ char_buf: &'b mut [u8; 4],
98
+ ) -> Result<Option<&'b [u8]>, R::Error> {
99
+ const MAX_NEEDLE_LEN: usize = 13;
100
+
101
+ match self.to_reconsume.take() {
102
+ Some(Some(x)) => {
103
+ return Ok(Some({
104
+ char_buf[0] = x;
105
+ &char_buf[..1]
106
+ }))
107
+ }
108
+ Some(None) => return Ok(None),
109
+ None => (),
110
+ }
111
+
112
+ let mut needle2 = [b'\0'; MAX_NEEDLE_LEN];
113
+ // Assert that we will have space for adding \r
114
+ // If not, just bump MAX_NEEDLE_LEN
115
+ debug_assert!(needle.len() < needle2.len());
116
+ needle2[..needle.len()].copy_from_slice(needle);
117
+ needle2[needle.len()] = b'\r';
118
+ let needle2_slice = &needle2[..=needle.len()];
119
+
120
+ let read = self.reader.read_until(needle2_slice, char_buf)?;
121
+ match read {
122
+ Some(b"\r") => {
123
+ self.last_character_was_cr = true;
124
+ char_validator.validate_byte(emitter, b'\n');
125
+ Ok(Some(b"\n"))
126
+ }
127
+ Some(mut xs) => {
128
+ char_validator.validate_bytes(emitter, xs);
129
+
130
+ if self.last_character_was_cr && xs.starts_with(b"\n") {
131
+ // TODO: we are scattering move_position in the read
132
+ // path across many places: read helper, tokenizer.rs,
133
+ // ... consolidate!
134
+ emitter.move_position(1);
135
+ xs = &xs[1..];
136
+ }
137
+
138
+ self.last_character_was_cr = false;
139
+ Ok(Some(xs))
140
+ }
141
+ None => {
142
+ self.last_character_was_cr = false;
143
+ Ok(None)
144
+ }
145
+ }
146
+ }
147
+
148
+ #[inline]
149
+ pub(crate) fn unread_byte(&mut self, c: Option<u8>) {
150
+ self.to_reconsume = Some(c);
151
+ }
152
+ }
153
+
154
+ /// A version of `match read_helper.read_char()` that "knows" about matched characters, so it can
155
+ /// produce a more efficient `read_until` call instead.
156
+ ///
157
+ /// An extremely limited subset of match patterns is supported.
158
+ ///
159
+ /// ```ignore
160
+ /// // I'm hitting multiple issues trying to get this test example to compile for doctests.
161
+ /// // Example: https://users.rust-lang.org/t/rustdoc-doctests-and-private-documentation/20955/6
162
+ /// use html5gum::{Reader, Tokenizer};
163
+ ///
164
+ /// fn before<R: Reader>(slf: &mut Tokenizer<R>) {
165
+ /// match slf.reader.read_char() {
166
+ /// Some("<") => todo!(),
167
+ /// Some(x) => todo!(),
168
+ /// None => todo!()
169
+ /// }
170
+ /// }
171
+ ///
172
+ /// fn after<R: Reader>(slf: &mut Tokenizer<R>) {
173
+ /// fast_read_char!(slf, emitter, match xs {
174
+ /// Some("<") => {
175
+ /// todo!()
176
+ /// }
177
+ /// Some(x) => {
178
+ /// todo!()
179
+ /// }
180
+ /// None => {
181
+ /// todo!()
182
+ /// }
183
+ /// })
184
+ /// }
185
+ /// ```
186
+ macro_rules! fast_read_char {
187
+ ($slf:expr, match $read_char:ident {
188
+ $(Some($($lit:literal)|*) => $arm:block)*
189
+ Some($xs:ident) => $catchall:block
190
+ None => $eof_catchall:block
191
+ }) => { loop {
192
+ let mut char_buf = [0; 4];
193
+ let $read_char = $crate::Tokenizer::read_until(
194
+ &mut $slf.reader,
195
+ &[ $($({
196
+ debug_assert_eq!($lit.len(), 1);
197
+ $lit[0]
198
+ }),*),* ],
199
+ &mut $slf.validator,
200
+ &mut $slf.emitter,
201
+ &mut char_buf,
202
+ )?;
203
+ break match $read_char {
204
+ $(
205
+ Some($($lit)|*) => {
206
+ $arm
207
+ }
208
+ )*
209
+ Some($xs) => {
210
+ // Prevent catch-all arm from using the machine_helper.
211
+ //
212
+ // State changes in catch-all arms are usually sign of a coding mistake. $xs
213
+ // may contain an arbitrary amount of characters, so it's more likely than not
214
+ // that the state is changed at the wrong read position.
215
+ //
216
+ // reconsume_in!() macro should not be used in this match arm either, as we can
217
+ // reconsume 2 characters at maximum, not a random $xs. Luckily that's kind of
218
+ // hard to do by accident.
219
+ #[allow(unused_variables)]
220
+ let _do_not_use = &mut $slf.machine_helper;
221
+ $catchall
222
+ }
223
+ None => {
224
+ $eof_catchall
225
+ }
226
+ };
227
+ } };
228
+ }
229
+
230
+ pub(crate) use fast_read_char;
231
+
232
+ macro_rules! slow_read_byte {
233
+ ($slf:expr, match c {
234
+ $($tt:tt)*
235
+ }) => {
236
+ loop {
237
+ break match $slf.read_byte()? {
238
+ $($tt)*
239
+ };
240
+ }
241
+ };
242
+ }
243
+
244
+ pub(crate) use slow_read_byte;