@dvaji/cheerox 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1094 -0
  3. package/Cargo.toml +46 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7579 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BS--qPzw.mjs +1648 -0
  18. package/dist/cheerio-compat-BS--qPzw.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DSwBAJ1s.cjs +1719 -0
  23. package/dist/cheerio-compat-DSwBAJ1s.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4119 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +26 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/package.json +167 -4
  167. package/scripts/check-native.mjs +15 -0
  168. package/scripts/check-release.mjs +234 -0
  169. package/scripts/fix-declarations.mjs +39 -0
  170. package/scripts/generate-notices.mjs +188 -0
  171. package/scripts/notices/README.md +13 -0
  172. package/scripts/notices/cheerio-MIT.txt +21 -0
  173. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  174. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  175. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  176. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  177. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  178. package/scripts/notices/napi-rs-MIT.txt +43 -0
  179. package/scripts/notices/neon-MIT.txt +19 -0
  180. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  181. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  182. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  183. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  184. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  185. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  186. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  187. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  188. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  189. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  190. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  191. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  192. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  193. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  194. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  195. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  196. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  197. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  198. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  199. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  200. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  201. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  202. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  203. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  204. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  205. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  206. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  207. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  208. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  209. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  210. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  211. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  212. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  213. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  214. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  215. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  216. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  217. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  218. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  219. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  220. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  221. package/scripts/notices/runtime.json +1632 -0
  222. package/scripts/notices/sources.json +561 -0
  223. package/scripts/notices/xray-MIT.txt +21 -0
  224. package/src/attribute-object.ts +71 -0
  225. package/src/attributes.ts +308 -0
  226. package/src/cheerio-compat/css.ts +58 -0
  227. package/src/cheerio-compat/render.ts +106 -0
  228. package/src/cheerio-compat/selectors.ts +51 -0
  229. package/src/cheerio-compat/types.ts +52 -0
  230. package/src/cheerio-compat.ts +2055 -0
  231. package/src/codec.ts +96 -0
  232. package/src/dom.ts +87 -0
  233. package/src/extract-plan.ts +97 -0
  234. package/src/extract.ts +322 -0
  235. package/src/hooks.ts +369 -0
  236. package/src/index.ts +48 -0
  237. package/src/io.ts +491 -0
  238. package/src/match.ts +207 -0
  239. package/src/modern.ts +259 -0
  240. package/src/native.ts +149 -0
  241. package/src/nodes.ts +831 -0
  242. package/src/parse.ts +53 -0
  243. package/src/parser-types.ts +157 -0
  244. package/src/router.ts +1129 -0
  245. package/src/selection.ts +682 -0
  246. package/src/slim.ts +12 -0
  247. package/src/snapshot.ts +291 -0
  248. package/src/transport.ts +113 -0
  249. package/src/types.ts +186 -0
  250. package/src/utils.ts +29 -0
  251. package/src/workers.ts +21 -0
  252. package/tsconfig.json +15 -0
  253. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,581 @@
1
+ //! Build htmlparser2-compatible trees from tokenizer events. Events come from
2
+ //! the native tokenizer port or from an explicitly supplied JavaScript
3
+ //! Tokenizer, whose recorded events directly determine the DOM.
4
+ use crate::arena::{
5
+ append_parsed, append_parsed_new, append_parsed_text, html_parser_void, implied_close,
6
+ DomArena, DomNode,
7
+ };
8
+ use crate::markup_tokenizer::Token;
9
+ use serde_json::{json, Value};
10
+ use std::borrow::Cow;
11
+
12
+ struct Pending {
13
+ name: String,
14
+ start: usize,
15
+ }
16
+ struct Builder<'a> {
17
+ arena: DomArena,
18
+ source: &'a Source<'a>,
19
+ parents: Vec<u32>,
20
+ foreign: Vec<bool>,
21
+ pending: Option<Pending>,
22
+ /// The pending tag's attributes in source order, collected so its map is
23
+ /// allocated once at its final size.
24
+ attributes: Vec<(String, String)>,
25
+ attribute_name: String,
26
+ attribute_value: String,
27
+ start: usize,
28
+ end: usize,
29
+ xml: bool,
30
+ lower_tags: bool,
31
+ lower_attrs: bool,
32
+ with_end_indices: bool,
33
+ recognize_self_closing: Option<bool>,
34
+ }
35
+ impl<'a> Builder<'a> {
36
+ fn new(source: &'a Source<'a>, options: Value) -> Self {
37
+ let xml = crate::arena::xml_mode(&options);
38
+ let lower_tags = options
39
+ .get("lowerCaseTags")
40
+ .and_then(Value::as_bool)
41
+ .unwrap_or(!xml);
42
+ let lower_attrs = options
43
+ .get("lowerCaseAttributeNames")
44
+ .and_then(Value::as_bool)
45
+ .unwrap_or(!xml);
46
+ let with_end_indices = options.get("withEndIndices") == Some(&Value::Bool(true));
47
+ let recognize_self_closing = options.get("recognizeSelfClosing").and_then(Value::as_bool);
48
+ let mut arena = DomArena::from_records(vec![DomNode::new("root")], options);
49
+ // Growing the arena one doubling at a time copies every node record
50
+ // again. Every node but text starts with `<`, and text lies between them.
51
+ let markup_starts = source
52
+ .units
53
+ .iter()
54
+ .filter(|unit| **unit == u16::from(b'<'))
55
+ .count();
56
+ arena.nodes.reserve(markup_starts + 1);
57
+ Self {
58
+ arena,
59
+ source,
60
+ parents: vec![0],
61
+ foreign: vec![xml],
62
+ pending: None,
63
+ attributes: Vec::new(),
64
+ attribute_name: String::new(),
65
+ attribute_value: String::new(),
66
+ start: 0,
67
+ end: 0,
68
+ xml,
69
+ lower_tags,
70
+ lower_attrs,
71
+ with_end_indices,
72
+ recognize_self_closing,
73
+ }
74
+ }
75
+ fn slice(&self, start: usize, end: usize) -> String {
76
+ self.source.slice(start, end)
77
+ }
78
+ fn name(&self, start: usize, end: usize) -> String {
79
+ let name = self.slice(start, end);
80
+ if self.lower_tags {
81
+ lowercase(name)
82
+ } else {
83
+ name
84
+ }
85
+ }
86
+ fn close(&mut self) {
87
+ if let Some(id) = (self.parents.len() > 1).then(|| self.parents.pop().unwrap()) {
88
+ if self.with_end_indices {
89
+ self.arena.nodes[id as usize]
90
+ .extras
91
+ .insert("endIndex".into(), json!(self.end));
92
+ }
93
+ }
94
+ }
95
+ fn open(&mut self, name: String) {
96
+ if !self.xml {
97
+ while self
98
+ .parents
99
+ .last()
100
+ .and_then(|id| self.arena.node(*id))
101
+ .and_then(|node| node.name.as_deref())
102
+ .is_some_and(|open| implied_close(&name, open))
103
+ {
104
+ self.close();
105
+ }
106
+ }
107
+ // htmlparser2 tracks foreign content only in HTML mode.
108
+ if !self.xml && !html_parser_void(&name) {
109
+ if matches!(name.as_str(), "svg" | "math") {
110
+ self.foreign.push(true);
111
+ } else if matches!(
112
+ name.as_str(),
113
+ "mi" | "mo"
114
+ | "mn"
115
+ | "ms"
116
+ | "mtext"
117
+ | "annotation-xml"
118
+ | "foreignobject"
119
+ | "desc"
120
+ | "title"
121
+ ) {
122
+ self.foreign.push(false);
123
+ }
124
+ }
125
+ self.attributes.clear();
126
+ self.pending = Some(Pending {
127
+ name,
128
+ start: self.start,
129
+ });
130
+ }
131
+ fn end_open(&mut self, self_closing: bool) {
132
+ if let Some(pending) = self.pending.take() {
133
+ let is_void = !self.xml && html_parser_void(&pending.name);
134
+ let kind = match pending.name.as_str() {
135
+ "script" if !self.xml => "script",
136
+ "style" if !self.xml => "style",
137
+ _ => "tag",
138
+ };
139
+ let lowercase_name = (pending.name.bytes().any(|byte| byte.is_ascii_uppercase()))
140
+ .then(|| pending.name.to_ascii_lowercase());
141
+ let id = append_parsed_new(
142
+ &mut self.arena,
143
+ *self.parents.last().unwrap(),
144
+ kind,
145
+ pending.start,
146
+ self.end,
147
+ );
148
+ let node = &mut self.arena.nodes[id as usize];
149
+ node.name = Some(pending.name);
150
+ node.attribs = deduplicated_attributes(self.attributes.drain(..));
151
+ let name = lowercase_name
152
+ .as_deref()
153
+ .or(self.arena.nodes[id as usize].name.as_deref())
154
+ .unwrap_or("");
155
+ let index = self.arena.name_index.get_or_insert_with(Default::default);
156
+ match index.get_mut(name) {
157
+ Some(ids) => ids.push(id),
158
+ None => {
159
+ index.insert(name.to_owned(), vec![id]);
160
+ }
161
+ }
162
+ if !is_void && !self_closing {
163
+ self.parents.push(id);
164
+ }
165
+ }
166
+ }
167
+ fn text(&mut self, data: String, end: usize) {
168
+ self.end = end.saturating_sub(1);
169
+ append_parsed_text(
170
+ &mut self.arena,
171
+ *self.parents.last().unwrap(),
172
+ data,
173
+ self.start,
174
+ self.end,
175
+ );
176
+ self.start = end;
177
+ }
178
+ fn instruction(&mut self, start: usize, end: usize, prefix: char) {
179
+ let value = self.slice(start, end);
180
+ let boundary = value
181
+ .find(|ch: char| ch.is_whitespace() || ch == '/')
182
+ .unwrap_or(value.len());
183
+ let name = if self.lower_tags {
184
+ value[..boundary].to_lowercase()
185
+ } else {
186
+ value[..boundary].to_owned()
187
+ };
188
+ let mut node = DomNode::new("directive");
189
+ node.name = Some(format!("{prefix}{name}"));
190
+ node.data = Some(format!("{prefix}{value}"));
191
+ append_parsed(
192
+ &mut self.arena,
193
+ *self.parents.last().unwrap(),
194
+ node,
195
+ self.start,
196
+ end,
197
+ );
198
+ self.end = end;
199
+ self.start = end + 1;
200
+ }
201
+ }
202
+
203
+ /// Attributes a tag listed in source order; a repeated attribute keeps its
204
+ /// first value, as htmlparser2 does. Few attributes are compared directly,
205
+ /// many through a set of the names seen.
206
+ fn deduplicated_attributes(
207
+ attributes: std::vec::Drain<'_, (String, String)>,
208
+ ) -> crate::arena::Attributes {
209
+ const DIRECT_LIMIT: usize = 16;
210
+ let count = attributes.len();
211
+ let mut result = crate::arena::Attributes::with_capacity(count);
212
+ if count <= DIRECT_LIMIT {
213
+ for (name, value) in attributes {
214
+ result.push_unique(name, Value::String(value));
215
+ }
216
+ return result;
217
+ }
218
+ let mut seen = std::collections::HashSet::with_capacity(count);
219
+ for (name, value) in attributes {
220
+ if seen.insert(name.clone()) {
221
+ result.push(name, Value::String(value));
222
+ }
223
+ }
224
+ result
225
+ }
226
+
227
+ /// Sources from JavaScript, or that contain codec frames, use the lossless
228
+ /// UTF-16 representation; others are plain Unicode.
229
+ fn is_codec_encoded(options: &Value) -> bool {
230
+ options.get("__utf16Codec") == Some(&Value::Bool(true))
231
+ || options.get("__losslessUnicode") == Some(&Value::Bool(true))
232
+ }
233
+
234
+ fn lowercase(mut name: String) -> String {
235
+ if name.is_ascii() {
236
+ name.make_ascii_lowercase();
237
+ name
238
+ } else {
239
+ name.to_lowercase()
240
+ }
241
+ }
242
+
243
+ /// The source as UTF-16 units, with each unit's byte offset in the original
244
+ /// string so spans can be copied from it instead of re-encoded.
245
+ struct Source<'a> {
246
+ text: &'a str,
247
+ units: Vec<u16>,
248
+ /// Byte offset per unit index, plus one for the end; `u32::MAX` marks an
249
+ /// index inside a character. `None` for ASCII, where offsets are indices.
250
+ offsets: Option<Vec<u32>>,
251
+ codec: bool,
252
+ }
253
+
254
+ struct IndexedUnits {
255
+ units: Vec<u16>,
256
+ offsets: Vec<u32>,
257
+ }
258
+
259
+ impl crate::codec::UnitSink for IndexedUnits {
260
+ fn ascii(&mut self, start: usize, run: &[u8]) {
261
+ self.units.extend(run.iter().map(|value| u16::from(*value)));
262
+ self.offsets
263
+ .extend((start..start + run.len()).map(|offset| offset as u32));
264
+ }
265
+ fn unit(&mut self, offset: Option<usize>, unit: u16) {
266
+ self.units.push(unit);
267
+ self.offsets
268
+ .push(offset.map_or(u32::MAX, |offset| offset as u32));
269
+ }
270
+ }
271
+
272
+ impl<'a> Source<'a> {
273
+ fn new(text: &'a str, options: &Value) -> Self {
274
+ let codec = is_codec_encoded(options);
275
+ if text.is_ascii() {
276
+ return Self {
277
+ text,
278
+ units: text.bytes().map(u16::from).collect(),
279
+ offsets: None,
280
+ codec,
281
+ };
282
+ }
283
+ let mut indexed = IndexedUnits {
284
+ units: Vec::with_capacity(text.len()),
285
+ offsets: Vec::with_capacity(text.len() + 1),
286
+ };
287
+ let framed = codec && crate::codec::has_encoding(text);
288
+ crate::codec::visit_units(text, framed, &mut indexed);
289
+ indexed.offsets.push(text.len() as u32);
290
+ Self {
291
+ text,
292
+ units: indexed.units,
293
+ offsets: Some(indexed.offsets),
294
+ codec,
295
+ }
296
+ }
297
+ fn borrowed_slice(&self, start: usize, end: usize) -> Option<&str> {
298
+ let start = start.min(self.units.len());
299
+ let end = end.clamp(start, self.units.len());
300
+ match self
301
+ .offsets
302
+ .as_ref()
303
+ .map(|offsets| (offsets[start], offsets[end]))
304
+ {
305
+ None => Some(&self.text[start..end]),
306
+ Some((from, to)) if from != u32::MAX && to != u32::MAX => {
307
+ Some(&self.text[from as usize..to as usize])
308
+ }
309
+ _ => None,
310
+ }
311
+ }
312
+ fn slice(&self, start: usize, end: usize) -> String {
313
+ if let Some(value) = self.borrowed_slice(start, end) {
314
+ return value.to_owned();
315
+ }
316
+ let start = start.min(self.units.len());
317
+ let end = end.clamp(start, self.units.len());
318
+ match self.codec {
319
+ true => crate::codec::from_units(&self.units[start..end]),
320
+ false => String::from_utf16_lossy(&self.units[start..end]),
321
+ }
322
+ }
323
+ }
324
+
325
+ /// Parses markup with the native port of htmlparser2's Tokenizer.
326
+ pub(crate) fn parse_markup(source: &str, options: Value) -> DomArena {
327
+ let source = Source::new(source, &options);
328
+ let decode = options.get("decodeEntities") != Some(&Value::Bool(false));
329
+ let mut builder = Builder::new(&source, options);
330
+ let xml = builder.xml;
331
+ crate::markup_tokenizer::tokenize(&source.units, xml, decode, |token| builder.handle(token));
332
+ builder.arena
333
+ }
334
+
335
+ /// Replays events recorded from a JavaScript htmlparser2 Tokenizer.
336
+ pub(crate) fn parse(source: &str, events: &Value, options: Value) -> DomArena {
337
+ let source = Source::new(source, &options);
338
+ let units = source.units.len() as i64;
339
+ let mut builder = Builder::new(&source, options);
340
+ for event in events.as_array().into_iter().flatten() {
341
+ let Some(tuple) = event.as_array() else {
342
+ continue;
343
+ };
344
+ let Some(event) = tuple.first().and_then(Value::as_str) else {
345
+ continue;
346
+ };
347
+ // htmlparser2 reports an unset section start as -1, which its
348
+ // `slice` resolves from the end of the input.
349
+ let arg = |index: usize| {
350
+ let value = tuple.get(index + 1).and_then(Value::as_i64).unwrap_or(0);
351
+ if value < 0 {
352
+ (units + value).max(0) as usize
353
+ } else {
354
+ value as usize
355
+ }
356
+ };
357
+ let (a, b, c) = (arg(0), arg(1), arg(2));
358
+ match event {
359
+ "ontext" => builder.on_text(a, b, false),
360
+ "ontextentity" => {
361
+ let text = builder.code_point(a);
362
+ builder.text(text, b);
363
+ }
364
+ "onopentagname" => builder.on_open_tag_name(a, b),
365
+ "onopentagend" => builder.on_open_tag_end(a),
366
+ "onselfclosingtag" => builder.on_self_closing_tag(a),
367
+ "onattribname" => builder.on_attribute_name(a, b),
368
+ "onattribdata" => builder.on_attribute_data(a, b, false),
369
+ "onattribentity" => {
370
+ let text = builder.code_point(a);
371
+ builder.attribute_value.push_str(&text);
372
+ }
373
+ "onattribend" => builder.on_attribute_end(b),
374
+ "onclosetag" => builder.on_close_tag(a, b),
375
+ "ondeclaration" => builder.instruction(a, b, '!'),
376
+ "onprocessinginstruction" => builder.instruction(a, b, '?'),
377
+ "oncomment" => builder.on_comment(a, b, c),
378
+ "oncdata" => builder.on_cdata(a, b, c),
379
+ "onend" => builder.on_end(),
380
+ _ => {}
381
+ }
382
+ }
383
+ builder.arena
384
+ }
385
+
386
+ impl Builder<'_> {
387
+ fn handle(&mut self, token: Token) {
388
+ match token {
389
+ Token::Text { start, end, decode } => self.on_text(start, end, decode),
390
+ Token::OpenTagName { start, end } => self.on_open_tag_name(start, end),
391
+ Token::OpenTagEnd { end } => self.on_open_tag_end(end),
392
+ Token::SelfClosingTag { end } => self.on_self_closing_tag(end),
393
+ Token::AttributeName { start, end } => self.on_attribute_name(start, end),
394
+ Token::AttributeData { start, end, decode } => {
395
+ self.on_attribute_data(start, end, decode)
396
+ }
397
+ Token::AttributeEnd { end } => self.on_attribute_end(end),
398
+ Token::CloseTag { start, end } => self.on_close_tag(start, end),
399
+ Token::Comment {
400
+ start,
401
+ end,
402
+ end_offset,
403
+ } => self.on_comment(start, end, end_offset),
404
+ Token::Cdata {
405
+ start,
406
+ end,
407
+ end_offset,
408
+ } => self.on_cdata(start, end, end_offset),
409
+ Token::Declaration { start, end } => self.instruction(start, end, '!'),
410
+ Token::ProcessingInstruction { start, end } => self.instruction(start, end, '?'),
411
+ Token::End => self.on_end(),
412
+ }
413
+ }
414
+ fn code_point(&self, code_point: usize) -> String {
415
+ if code_point <= 0xffff {
416
+ crate::codec::from_units(&[code_point as u16])
417
+ } else {
418
+ crate::codec::encode_scalar(char::from_u32(code_point as u32).unwrap_or('\u{fffd}'))
419
+ }
420
+ }
421
+ fn on_text(&mut self, start: usize, end: usize, decode: bool) {
422
+ let raw = self.slice(start, end);
423
+ let text = if decode && raw.contains('&') {
424
+ crate::arena::decode_entities_mode(&raw, self.xml, false, self.source.codec)
425
+ } else {
426
+ raw
427
+ };
428
+ self.text(text, end);
429
+ }
430
+ fn on_open_tag_name(&mut self, start: usize, end: usize) {
431
+ self.end = end;
432
+ let name = self.name(start, end);
433
+ self.open(name);
434
+ }
435
+ fn on_open_tag_end(&mut self, end: usize) {
436
+ self.end = end;
437
+ self.end_open(false);
438
+ self.start = end + 1;
439
+ }
440
+ fn on_self_closing_tag(&mut self, end: usize) {
441
+ self.end = end;
442
+ let self_closing = self.recognize_self_closing.unwrap_or(self.xml)
443
+ || *self.foreign.last().unwrap_or(&self.xml);
444
+ self.end_open(self_closing);
445
+ self.start = end + 1;
446
+ }
447
+ fn on_attribute_name(&mut self, start: usize, end: usize) {
448
+ self.start = start;
449
+ let name = self.slice(start, end);
450
+ self.attribute_name = if self.lower_attrs {
451
+ lowercase(name)
452
+ } else {
453
+ name
454
+ };
455
+ }
456
+ fn on_attribute_data(&mut self, start: usize, end: usize, decode: bool) {
457
+ let raw = self.slice(start, end);
458
+ let data = if decode && raw.contains('&') {
459
+ crate::arena::decode_entities_mode(&raw, self.xml, true, self.source.codec)
460
+ } else {
461
+ raw
462
+ };
463
+ if self.attribute_value.is_empty() {
464
+ self.attribute_value = data;
465
+ } else {
466
+ self.attribute_value.push_str(&data);
467
+ }
468
+ }
469
+ fn on_attribute_end(&mut self, end: usize) {
470
+ self.end = end;
471
+ let value = std::mem::take(&mut self.attribute_value);
472
+ if self.pending.is_some() {
473
+ self.attributes
474
+ .push((std::mem::take(&mut self.attribute_name), value));
475
+ }
476
+ }
477
+ fn on_close_tag(&mut self, start: usize, end: usize) {
478
+ self.end = end;
479
+ let source = self.source;
480
+ let name: Cow<'_, str> = match source.borrowed_slice(start, end) {
481
+ Some(name)
482
+ if !self.lower_tags
483
+ || (name.is_ascii() && !name.bytes().any(|byte| byte.is_ascii_uppercase())) =>
484
+ {
485
+ Cow::Borrowed(name)
486
+ }
487
+ _ => Cow::Owned(self.name(start, end)),
488
+ };
489
+ if !self.xml
490
+ && matches!(
491
+ name.as_ref(),
492
+ "svg"
493
+ | "math"
494
+ | "mi"
495
+ | "mo"
496
+ | "mn"
497
+ | "ms"
498
+ | "mtext"
499
+ | "annotation-xml"
500
+ | "foreignobject"
501
+ | "desc"
502
+ | "title"
503
+ )
504
+ {
505
+ self.foreign.pop();
506
+ }
507
+ if !(!self.xml && html_parser_void(&name)) {
508
+ if let Some(position) = self.parents.iter().rposition(|id| {
509
+ self.arena.node(*id).and_then(|node| node.name.as_deref()) == Some(name.as_ref())
510
+ }) {
511
+ while self.parents.len() > position {
512
+ self.close();
513
+ }
514
+ } else if !self.xml && name == "p" {
515
+ self.open(name.into_owned());
516
+ self.end_open(true);
517
+ }
518
+ } else if !self.xml && name == "br" {
519
+ self.open(name.into_owned());
520
+ self.end_open(true);
521
+ }
522
+ self.start = end + 1;
523
+ }
524
+ fn on_comment(&mut self, start: usize, end: usize, end_offset: usize) {
525
+ let mut node = DomNode::new("comment");
526
+ node.data = Some(self.slice(start, end.saturating_sub(end_offset)));
527
+ append_parsed(
528
+ &mut self.arena,
529
+ *self.parents.last().unwrap(),
530
+ node,
531
+ self.start,
532
+ end,
533
+ );
534
+ self.end = end;
535
+ self.start = end + 1;
536
+ }
537
+ fn on_cdata(&mut self, start: usize, end: usize, end_offset: usize) {
538
+ let data = self.slice(start, end.saturating_sub(end_offset));
539
+ if self.xml || self.arena.options.get("recognizeCDATA") == Some(&Value::Bool(true)) {
540
+ let id = append_parsed(
541
+ &mut self.arena,
542
+ *self.parents.last().unwrap(),
543
+ DomNode::new("cdata"),
544
+ self.start,
545
+ end,
546
+ );
547
+ let mut text = DomNode::new("text");
548
+ text.data = Some(data);
549
+ text.parent = Some(id);
550
+ text.extras.insert("startIndex".into(), Value::Null);
551
+ text.extras.insert(
552
+ "endIndex".into(),
553
+ if self.arena.options.get("withEndIndices") == Some(&Value::Bool(true)) {
554
+ json!(end)
555
+ } else {
556
+ Value::Null
557
+ },
558
+ );
559
+ let child = self.arena.push(text);
560
+ self.arena.nodes[id as usize].children.push(child);
561
+ } else {
562
+ let mut node = DomNode::new("comment");
563
+ node.data = Some(format!("[CDATA[{data}]]"));
564
+ append_parsed(
565
+ &mut self.arena,
566
+ *self.parents.last().unwrap(),
567
+ node,
568
+ self.start,
569
+ end,
570
+ );
571
+ }
572
+ self.end = end;
573
+ self.start = end + 1;
574
+ }
575
+ fn on_end(&mut self) {
576
+ self.end = self.start;
577
+ while self.parents.len() > 1 {
578
+ self.close();
579
+ }
580
+ }
581
+ }