@dvaji/cheerox 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1091 -0
  3. package/Cargo.toml +47 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7581 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BKkfJEAr.mjs +1648 -0
  18. package/dist/cheerio-compat-BKkfJEAr.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DcnYK9OB.cjs +1719 -0
  23. package/dist/cheerio-compat-DcnYK9OB.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4120 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +35 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/native/vendor/scraper-0.24.0/Cargo.toml +52 -0
  167. package/native/vendor/scraper-0.24.0/LICENSE +15 -0
  168. package/native/vendor/scraper-0.24.0/src/element_ref/element.rs +212 -0
  169. package/native/vendor/scraper-0.24.0/src/element_ref/mod.rs +236 -0
  170. package/native/vendor/scraper-0.24.0/src/element_ref/serializable.rs +15 -0
  171. package/native/vendor/scraper-0.24.0/src/error/utils.rs +91 -0
  172. package/native/vendor/scraper-0.24.0/src/error.rs +118 -0
  173. package/native/vendor/scraper-0.24.0/src/html/mod.rs +270 -0
  174. package/native/vendor/scraper-0.24.0/src/html/serializable.rs +27 -0
  175. package/native/vendor/scraper-0.24.0/src/html/tree_sink.rs +285 -0
  176. package/native/vendor/scraper-0.24.0/src/lib.rs +180 -0
  177. package/native/vendor/scraper-0.24.0/src/node/serializable.rs +52 -0
  178. package/native/vendor/scraper-0.24.0/src/node.rs +465 -0
  179. package/native/vendor/scraper-0.24.0/src/selectable.rs +70 -0
  180. package/native/vendor/scraper-0.24.0/src/selector.rs +290 -0
  181. package/package.json +167 -4
  182. package/scripts/check-native.mjs +15 -0
  183. package/scripts/check-release.mjs +234 -0
  184. package/scripts/fix-declarations.mjs +39 -0
  185. package/scripts/generate-notices.mjs +194 -0
  186. package/scripts/notices/README.md +13 -0
  187. package/scripts/notices/cheerio-MIT.txt +21 -0
  188. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  189. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  190. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  191. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  192. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  193. package/scripts/notices/napi-rs-MIT.txt +43 -0
  194. package/scripts/notices/neon-MIT.txt +19 -0
  195. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  196. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  197. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  198. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  199. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  200. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  201. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  202. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  203. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  204. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  205. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  206. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  207. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  208. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  209. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  210. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  211. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  212. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  213. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  214. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  215. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  216. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  217. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  218. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  219. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  220. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  221. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  222. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  223. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  224. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  225. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  226. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  227. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  228. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  229. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  230. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  231. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  232. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  233. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  234. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  235. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  236. package/scripts/notices/runtime.json +1632 -0
  237. package/scripts/notices/sources.json +561 -0
  238. package/scripts/notices/xray-MIT.txt +21 -0
  239. package/src/attribute-object.ts +71 -0
  240. package/src/attributes.ts +308 -0
  241. package/src/cheerio-compat/css.ts +58 -0
  242. package/src/cheerio-compat/render.ts +106 -0
  243. package/src/cheerio-compat/selectors.ts +51 -0
  244. package/src/cheerio-compat/types.ts +52 -0
  245. package/src/cheerio-compat.ts +2055 -0
  246. package/src/codec.ts +96 -0
  247. package/src/dom.ts +87 -0
  248. package/src/extract-plan.ts +97 -0
  249. package/src/extract.ts +322 -0
  250. package/src/hooks.ts +369 -0
  251. package/src/index.ts +48 -0
  252. package/src/io.ts +491 -0
  253. package/src/match.ts +207 -0
  254. package/src/modern.ts +259 -0
  255. package/src/native.ts +149 -0
  256. package/src/nodes.ts +831 -0
  257. package/src/parse.ts +53 -0
  258. package/src/parser-types.ts +157 -0
  259. package/src/router.ts +1129 -0
  260. package/src/selection.ts +682 -0
  261. package/src/slim.ts +12 -0
  262. package/src/snapshot.ts +291 -0
  263. package/src/transport.ts +113 -0
  264. package/src/types.ts +186 -0
  265. package/src/utils.ts +29 -0
  266. package/src/workers.ts +21 -0
  267. package/tsconfig.json +15 -0
  268. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,530 @@
1
+ // Copyright 2014-2017 The html5ever Project Developers. See the
2
+ // COPYRIGHT file at the top-level directory of this distribution.
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5
+ // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6
+ // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7
+ // option. This file may not be copied, modified, or distributed
8
+ // except according to those terms.
9
+
10
+ use super::{TokenSink, Tokenizer};
11
+ use crate::buffer_queue::BufferQueue;
12
+ use crate::data;
13
+ use crate::tendril::StrTendril;
14
+
15
+ use log::debug;
16
+ use std::borrow::Cow::{self, Borrowed};
17
+ use std::char::from_u32;
18
+
19
+ use self::State::*;
20
+ pub(super) use self::Status::*;
21
+
22
+ //§ tokenizing-character-references
23
+ /// Longest reference body plus semicolon that `resolve_terminated_reference`
24
+ /// looks for; the longest named reference has 32 characters.
25
+ const MAXIMUM_TERMINATED_REFERENCE_LENGTH: usize = 40;
26
+
27
+ pub(super) struct CharRef {
28
+ /// The resulting character(s)
29
+ pub(super) chars: [char; 2],
30
+
31
+ /// How many slots in `chars` are valid?
32
+ pub(super) num_chars: u8,
33
+ }
34
+
35
+ pub(super) enum Status {
36
+ Stuck,
37
+ Progress,
38
+ Done,
39
+ }
40
+
41
+ #[derive(Debug)]
42
+ enum State {
43
+ Begin,
44
+ Octothorpe,
45
+ Numeric(u32), // base
46
+ NumericSemicolon,
47
+ Named,
48
+ BogusName,
49
+ }
50
+
51
+ pub(super) struct CharRefTokenizer {
52
+ state: State,
53
+ result: Option<CharRef>,
54
+ is_consumed_in_attribute: bool,
55
+
56
+ num: u32,
57
+ num_too_big: bool,
58
+ seen_digit: bool,
59
+ hex_marker: Option<char>,
60
+
61
+ name_buf_opt: Option<StrTendril>,
62
+ name_match: Option<(u32, u32)>,
63
+ name_len: usize,
64
+ }
65
+
66
+ impl CharRefTokenizer {
67
+ pub(super) fn new(is_consumed_in_attribute: bool) -> CharRefTokenizer {
68
+ CharRefTokenizer {
69
+ is_consumed_in_attribute,
70
+ state: Begin,
71
+ result: None,
72
+ num: 0,
73
+ num_too_big: false,
74
+ seen_digit: false,
75
+ hex_marker: None,
76
+ name_buf_opt: None,
77
+ name_match: None,
78
+ name_len: 0,
79
+ }
80
+ }
81
+
82
+ // A CharRefTokenizer can only tokenize one character reference,
83
+ // so this method consumes the tokenizer.
84
+ pub(super) fn get_result(self) -> CharRef {
85
+ self.result.expect("get_result called before done")
86
+ }
87
+
88
+ fn name_buf(&self) -> &StrTendril {
89
+ self.name_buf_opt
90
+ .as_ref()
91
+ .expect("name_buf missing in named character reference")
92
+ }
93
+
94
+ fn name_buf_mut(&mut self) -> &mut StrTendril {
95
+ self.name_buf_opt
96
+ .as_mut()
97
+ .expect("name_buf missing in named character reference")
98
+ }
99
+
100
+ fn finish_none(&mut self) -> Status {
101
+ self.result = Some(CharRef {
102
+ chars: ['\0', '\0'],
103
+ num_chars: 0,
104
+ });
105
+ Done
106
+ }
107
+
108
+ fn finish_one(&mut self, c: char) -> Status {
109
+ self.result = Some(CharRef {
110
+ chars: [c, '\0'],
111
+ num_chars: 1,
112
+ });
113
+ Done
114
+ }
115
+ }
116
+
117
+ impl CharRefTokenizer {
118
+ pub(super) fn step<Sink: TokenSink>(
119
+ &mut self,
120
+ tokenizer: &Tokenizer<Sink>,
121
+ input: &BufferQueue,
122
+ ) -> Status {
123
+ if self.result.is_some() {
124
+ return Done;
125
+ }
126
+
127
+ debug!("char ref tokenizer stepping in state {:?}", self.state);
128
+ match self.state {
129
+ Begin => self.do_begin(tokenizer, input),
130
+ Octothorpe => self.do_octothorpe(tokenizer, input),
131
+ Numeric(base) => self.do_numeric(tokenizer, input, base),
132
+ NumericSemicolon => self.do_numeric_semicolon(tokenizer, input),
133
+ Named => self.do_named(tokenizer, input),
134
+ BogusName => self.do_bogus_name(tokenizer, input),
135
+ }
136
+ }
137
+
138
+ fn do_begin<Sink: TokenSink>(
139
+ &mut self,
140
+ tokenizer: &Tokenizer<Sink>,
141
+ input: &BufferQueue,
142
+ ) -> Status {
143
+ if let Some(status) = self.resolve_terminated_reference(tokenizer, input) {
144
+ return status;
145
+ }
146
+ match tokenizer.peek(input) {
147
+ Some('a'..='z' | 'A'..='Z' | '0'..='9') => {
148
+ self.state = Named;
149
+ self.name_buf_opt = Some(StrTendril::new());
150
+ Progress
151
+ }
152
+ Some('#') => {
153
+ tokenizer.discard_char(input);
154
+ self.state = Octothorpe;
155
+ Progress
156
+ }
157
+ Some(_) => self.finish_none(),
158
+ None => Stuck,
159
+ }
160
+ }
161
+
162
+ /// Resolves a reference that ends with a semicolon within the next input
163
+ /// chunk in one step. Stepping character by character costs a state
164
+ /// machine round trip and a table lookup per character, which dominates
165
+ /// pages full of references such as `&nbsp;`. Any other input takes the
166
+ /// character states, which reach the same result for these references.
167
+ fn resolve_terminated_reference<Sink: TokenSink>(
168
+ &mut self,
169
+ tokenizer: &Tokenizer<Sink>,
170
+ input: &BufferQueue,
171
+ ) -> Option<Status> {
172
+ if tokenizer.reconsume.get() {
173
+ return None;
174
+ }
175
+ let mut chunk = input.peek_front_chunk_mut()?;
176
+ let bytes = chunk.as_bytes();
177
+ let semicolon = bytes
178
+ .iter()
179
+ .take(MAXIMUM_TERMINATED_REFERENCE_LENGTH)
180
+ .position(|&byte| byte == b';')?;
181
+ let body = &bytes[..semicolon];
182
+ let resolved = match body {
183
+ [b'#', b'x' | b'X', digits @ ..] => self.accumulate_number(digits, 16),
184
+ [b'#', digits @ ..] => self.accumulate_number(digits, 10),
185
+ [first, ..] if first.is_ascii_alphabetic() && body.iter().all(u8::is_ascii_alphanumeric) => {
186
+ match data::NAMED_ENTITIES.get(&chunk[..=semicolon]) {
187
+ Some(&(first, second)) if first != 0 => {
188
+ self.result = Some(CharRef {
189
+ chars: [from_u32(first).unwrap(), from_u32(second).unwrap()],
190
+ num_chars: if second == 0 { 1 } else { 2 },
191
+ });
192
+ true
193
+ },
194
+ _ => false,
195
+ }
196
+ },
197
+ _ => false,
198
+ };
199
+ if !resolved {
200
+ return None;
201
+ }
202
+ chunk.pop_front(semicolon as u32 + 1);
203
+ let is_chunk_consumed = chunk.is_empty();
204
+ drop(chunk);
205
+ if is_chunk_consumed {
206
+ input.pop_front();
207
+ }
208
+ tokenizer.ignore_lf.set(false);
209
+ if self.result.is_some() {
210
+ return Some(Done);
211
+ }
212
+ Some(self.finish_numeric(tokenizer))
213
+ }
214
+
215
+ /// Accumulates `digits` as `do_numeric` does, if they are all digits.
216
+ fn accumulate_number(&mut self, digits: &[u8], base: u32) -> bool {
217
+ let is_number = !digits.is_empty()
218
+ && digits
219
+ .iter()
220
+ .all(|&digit| char::from(digit).is_digit(base));
221
+ if !is_number {
222
+ return false;
223
+ }
224
+ for &digit in digits {
225
+ self.num = self.num.wrapping_mul(base);
226
+ if self.num > 0x10FFFF {
227
+ self.num_too_big = true;
228
+ }
229
+ self.num = self.num.wrapping_add(char::from(digit).to_digit(base).unwrap());
230
+ }
231
+ true
232
+ }
233
+
234
+ fn do_octothorpe<Sink: TokenSink>(
235
+ &mut self,
236
+ tokenizer: &Tokenizer<Sink>,
237
+ input: &BufferQueue,
238
+ ) -> Status {
239
+ match tokenizer.peek(input) {
240
+ Some(c @ ('x' | 'X')) => {
241
+ tokenizer.discard_char(input);
242
+ self.hex_marker = Some(c);
243
+ self.state = Numeric(16);
244
+ }
245
+ Some(_) => {
246
+ self.hex_marker = None;
247
+ self.state = Numeric(10);
248
+ }
249
+ None => return Stuck,
250
+ }
251
+ Progress
252
+ }
253
+
254
+ fn do_numeric<Sink: TokenSink>(
255
+ &mut self,
256
+ tokenizer: &Tokenizer<Sink>,
257
+ input: &BufferQueue,
258
+ base: u32,
259
+ ) -> Status {
260
+ let Some(c) = tokenizer.peek(input) else {
261
+ return Stuck;
262
+ };
263
+ match c.to_digit(base) {
264
+ Some(n) => {
265
+ tokenizer.discard_char(input);
266
+ self.num = self.num.wrapping_mul(base);
267
+ if self.num > 0x10FFFF {
268
+ // We might overflow, and the character is definitely invalid.
269
+ // We still parse digits and semicolon, but don't use the result.
270
+ self.num_too_big = true;
271
+ }
272
+ self.num = self.num.wrapping_add(n);
273
+ self.seen_digit = true;
274
+ Progress
275
+ }
276
+
277
+ None if !self.seen_digit => self.unconsume_numeric(tokenizer, input),
278
+
279
+ None => {
280
+ self.state = NumericSemicolon;
281
+ Progress
282
+ }
283
+ }
284
+ }
285
+
286
+ fn do_numeric_semicolon<Sink: TokenSink>(
287
+ &mut self,
288
+ tokenizer: &Tokenizer<Sink>,
289
+ input: &BufferQueue,
290
+ ) -> Status {
291
+ match tokenizer.peek(input) {
292
+ Some(';') => tokenizer.discard_char(input),
293
+ Some(_) => tokenizer.emit_error(Borrowed(
294
+ "Semicolon missing after numeric character reference",
295
+ )),
296
+ None => return Stuck,
297
+ };
298
+ self.finish_numeric(tokenizer)
299
+ }
300
+
301
+ fn unconsume_numeric<Sink: TokenSink>(
302
+ &mut self,
303
+ tokenizer: &Tokenizer<Sink>,
304
+ input: &BufferQueue,
305
+ ) -> Status {
306
+ let mut unconsume = StrTendril::from_char('#');
307
+ if let Some(c) = self.hex_marker {
308
+ unconsume.push_char(c)
309
+ }
310
+
311
+ input.push_front(unconsume);
312
+ tokenizer.emit_error(Borrowed("Numeric character reference without digits"));
313
+ self.finish_none()
314
+ }
315
+
316
+ fn finish_numeric<Sink: TokenSink>(&mut self, tokenizer: &Tokenizer<Sink>) -> Status {
317
+ fn conv(n: u32) -> char {
318
+ from_u32(n).expect("invalid char missed by error handling cases")
319
+ }
320
+
321
+ let (c, error) = match self.num {
322
+ n if (n > 0x10FFFF) || self.num_too_big => ('\u{fffd}', true),
323
+ 0x00 | 0xD800..=0xDFFF => ('\u{fffd}', true),
324
+
325
+ 0x80..=0x9F => match data::C1_REPLACEMENTS[(self.num - 0x80) as usize] {
326
+ Some(c) => (c, true),
327
+ None => (conv(self.num), true),
328
+ },
329
+
330
+ 0x01..=0x08 | 0x0B | 0x0D..=0x1F | 0x7F | 0xFDD0..=0xFDEF => (conv(self.num), true),
331
+
332
+ n if (n & 0xFFFE) == 0xFFFE => (conv(n), true),
333
+
334
+ n => (conv(n), false),
335
+ };
336
+
337
+ if error {
338
+ let msg = if tokenizer.opts.exact_errors {
339
+ Cow::from(format!(
340
+ "Invalid numeric character reference value 0x{:06X}",
341
+ self.num
342
+ ))
343
+ } else {
344
+ Cow::from("Invalid numeric character reference")
345
+ };
346
+ tokenizer.emit_error(msg);
347
+ }
348
+
349
+ self.finish_one(c)
350
+ }
351
+
352
+ fn do_named<Sink: TokenSink>(
353
+ &mut self,
354
+ tokenizer: &Tokenizer<Sink>,
355
+ input: &BufferQueue,
356
+ ) -> Status {
357
+ // peek + discard skips over newline normalization, therefore making it easier to
358
+ // un-consume
359
+ let Some(c) = tokenizer.peek(input) else {
360
+ return Stuck;
361
+ };
362
+ tokenizer.discard_char(input);
363
+ self.name_buf_mut().push_char(c);
364
+ match data::NAMED_ENTITIES.get(&self.name_buf()[..]) {
365
+ // We have either a full match or a prefix of one.
366
+ Some(&m) => {
367
+ if m.0 != 0 {
368
+ // We have a full match, but there might be a longer one to come.
369
+ self.name_match = Some(m);
370
+ self.name_len = self.name_buf().len();
371
+ }
372
+ // Otherwise we just have a prefix match.
373
+ Progress
374
+ }
375
+
376
+ // Can't continue the match.
377
+ None => self.finish_named(tokenizer, input, Some(c)),
378
+ }
379
+ }
380
+
381
+ fn emit_name_error<Sink: TokenSink>(&mut self, tokenizer: &Tokenizer<Sink>) {
382
+ let msg = if tokenizer.opts.exact_errors {
383
+ Cow::from(format!("Invalid character reference &{}", self.name_buf()))
384
+ } else {
385
+ Cow::from("Invalid character reference")
386
+ };
387
+ tokenizer.emit_error(msg);
388
+ }
389
+
390
+ fn unconsume_name(&mut self, input: &BufferQueue) {
391
+ input.push_front(self.name_buf_opt.take().unwrap());
392
+ }
393
+
394
+ fn finish_named<Sink: TokenSink>(
395
+ &mut self,
396
+ tokenizer: &Tokenizer<Sink>,
397
+ input: &BufferQueue,
398
+ end_char: Option<char>,
399
+ ) -> Status {
400
+ match self.name_match {
401
+ None => {
402
+ match end_char {
403
+ Some(c) if c.is_ascii_alphanumeric() => {
404
+ // Keep looking for a semicolon, to determine whether
405
+ // we emit a parse error.
406
+ self.state = BogusName;
407
+ return Progress;
408
+ }
409
+
410
+ // Check length because &; is not a parse error.
411
+ Some(';') if self.name_buf().len() > 1 => self.emit_name_error(tokenizer),
412
+
413
+ _ => (),
414
+ }
415
+ self.unconsume_name(input);
416
+ self.finish_none()
417
+ }
418
+
419
+ Some((c1, c2)) => {
420
+ // We have a complete match, but we may have consumed
421
+ // additional characters into self.name_buf. Usually
422
+ // at least one, but several in cases like
423
+ //
424
+ // &not => match for U+00AC
425
+ // &noti => valid prefix for &notin
426
+ // &notit => can't continue match
427
+
428
+ let name_len = self.name_len;
429
+ assert!(name_len > 0);
430
+ let last_matched = self.name_buf()[name_len - 1..].chars().next().unwrap();
431
+
432
+ // There might not be a next character after the match, if
433
+ // we had a full match and then hit EOF.
434
+ let next_after = if name_len == self.name_buf().len() {
435
+ None
436
+ } else {
437
+ Some(self.name_buf()[name_len..].chars().next().unwrap())
438
+ };
439
+
440
+ // If the character reference was consumed as part of an attribute, and the last
441
+ // character matched is not a U+003B SEMICOLON character (;), and the next input
442
+ // character is either a U+003D EQUALS SIGN character (=) or an ASCII alphanumeric,
443
+ // then, for historical reasons, flush code points consumed as a character
444
+ // reference and switch to the return state.
445
+
446
+ let unconsume_all = match (self.is_consumed_in_attribute, last_matched, next_after)
447
+ {
448
+ (_, ';', _) => false,
449
+ (true, _, Some('=')) => true,
450
+ (true, _, Some(c)) if c.is_ascii_alphanumeric() => true,
451
+ _ => {
452
+ // 1. If the last character matched is not a U+003B SEMICOLON character
453
+ // (;), then this is a missing-semicolon-after-character-reference parse
454
+ // error.
455
+ tokenizer.emit_error(Borrowed(
456
+ "Character reference does not end with semicolon",
457
+ ));
458
+ false
459
+ }
460
+ };
461
+
462
+ if unconsume_all {
463
+ self.unconsume_name(input);
464
+ self.finish_none()
465
+ } else {
466
+ input.push_front(StrTendril::from_slice(&self.name_buf()[name_len..]));
467
+ tokenizer.ignore_lf.set(false);
468
+ self.result = Some(CharRef {
469
+ chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()],
470
+ num_chars: if c2 == 0 { 1 } else { 2 },
471
+ });
472
+ Done
473
+ }
474
+ }
475
+ }
476
+ }
477
+
478
+ fn do_bogus_name<Sink: TokenSink>(
479
+ &mut self,
480
+ tokenizer: &Tokenizer<Sink>,
481
+ input: &BufferQueue,
482
+ ) -> Status {
483
+ // peek + discard skips over newline normalization, therefore making it easier to
484
+ // un-consume
485
+ let Some(c) = tokenizer.peek(input) else {
486
+ return Stuck;
487
+ };
488
+ tokenizer.discard_char(input);
489
+ self.name_buf_mut().push_char(c);
490
+ match c {
491
+ _ if c.is_ascii_alphanumeric() => return Progress,
492
+ ';' => self.emit_name_error(tokenizer),
493
+ _ => (),
494
+ }
495
+ self.unconsume_name(input);
496
+ self.finish_none()
497
+ }
498
+
499
+ pub(super) fn end_of_file<Sink: TokenSink>(
500
+ &mut self,
501
+ tokenizer: &Tokenizer<Sink>,
502
+ input: &BufferQueue,
503
+ ) {
504
+ while self.result.is_none() {
505
+ match self.state {
506
+ Begin => drop(self.finish_none()),
507
+
508
+ Numeric(_) if !self.seen_digit => drop(self.unconsume_numeric(tokenizer, input)),
509
+
510
+ Numeric(_) | NumericSemicolon => {
511
+ tokenizer.emit_error(Borrowed("EOF in numeric character reference"));
512
+ self.finish_numeric(tokenizer);
513
+ }
514
+
515
+ Named => drop(self.finish_named(tokenizer, input, None)),
516
+
517
+ BogusName => {
518
+ self.unconsume_name(input);
519
+ self.finish_none();
520
+ }
521
+
522
+ Octothorpe => {
523
+ input.push_front(StrTendril::from_slice("#"));
524
+ tokenizer.emit_error(Borrowed("EOF after '#' in character reference"));
525
+ self.finish_none();
526
+ }
527
+ }
528
+ }
529
+ }
530
+ }
@@ -0,0 +1,99 @@
1
+ // Copyright 2014-2017 The html5ever Project Developers. See the
2
+ // COPYRIGHT file at the top-level directory of this distribution.
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5
+ // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6
+ // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7
+ // option. This file may not be copied, modified, or distributed
8
+ // except according to those terms.
9
+
10
+ use crate::interface::Attribute;
11
+ use crate::tendril::StrTendril;
12
+ use crate::tokenizer::states;
13
+ use crate::LocalName;
14
+ use std::borrow::Cow;
15
+
16
+ pub use self::TagKind::{EndTag, StartTag};
17
+ pub use self::Token::{CharacterTokens, CommentToken, DoctypeToken, TagToken};
18
+ pub use self::Token::{EOFToken, NullCharacterToken, ParseError};
19
+
20
+ /// A `DOCTYPE` token.
21
+ // FIXME: already exists in Servo DOM
22
+ #[derive(PartialEq, Eq, Clone, Debug, Default)]
23
+ pub struct Doctype {
24
+ pub name: Option<StrTendril>,
25
+ pub public_id: Option<StrTendril>,
26
+ pub system_id: Option<StrTendril>,
27
+ pub force_quirks: bool,
28
+ }
29
+
30
+ #[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)]
31
+ pub enum TagKind {
32
+ StartTag,
33
+ EndTag,
34
+ }
35
+
36
+ /// A tag token.
37
+ #[derive(PartialEq, Eq, Clone, Debug)]
38
+ pub struct Tag {
39
+ pub kind: TagKind,
40
+ pub name: LocalName,
41
+ pub self_closing: bool,
42
+ pub attrs: Vec<Attribute>,
43
+ }
44
+
45
+ impl Tag {
46
+ /// Are the tags equivalent when we don't care about attribute order?
47
+ /// Also ignores the self-closing flag.
48
+ pub fn equiv_modulo_attr_order(&self, other: &Tag) -> bool {
49
+ if (self.kind != other.kind) || (self.name != other.name) {
50
+ return false;
51
+ }
52
+
53
+ let mut self_attrs = self.attrs.clone();
54
+ let mut other_attrs = other.attrs.clone();
55
+ self_attrs.sort();
56
+ other_attrs.sort();
57
+
58
+ self_attrs == other_attrs
59
+ }
60
+ }
61
+
62
+ #[derive(PartialEq, Eq, Debug)]
63
+ pub enum Token {
64
+ DoctypeToken(Doctype),
65
+ TagToken(Tag),
66
+ CommentToken(StrTendril),
67
+ CharacterTokens(StrTendril),
68
+ NullCharacterToken,
69
+ EOFToken,
70
+ ParseError(Cow<'static, str>),
71
+ }
72
+
73
+ #[derive(Debug, PartialEq)]
74
+ #[must_use]
75
+ pub enum TokenSinkResult<Handle> {
76
+ Continue,
77
+ Script(Handle),
78
+ Plaintext,
79
+ RawData(states::RawKind),
80
+ }
81
+
82
+ /// Types which can receive tokens from the tokenizer.
83
+ pub trait TokenSink {
84
+ type Handle;
85
+
86
+ /// Process a token.
87
+ fn process_token(&self, token: Token, line_number: u64) -> TokenSinkResult<Self::Handle>;
88
+
89
+ // Signal sink that tokenization reached the end.
90
+ fn end(&self) {}
91
+
92
+ /// Used in the markup declaration open state. By default, this always
93
+ /// returns false and thus all CDATA sections are tokenized as bogus
94
+ /// comments.
95
+ /// <https://html.spec.whatwg.org/multipage/#markup-declaration-open-state>
96
+ fn adjusted_current_node_present_but_not_in_html_namespace(&self) -> bool {
97
+ false
98
+ }
99
+ }