@dvaji/cheerox 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1091 -0
  3. package/Cargo.toml +47 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7581 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BKkfJEAr.mjs +1648 -0
  18. package/dist/cheerio-compat-BKkfJEAr.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DcnYK9OB.cjs +1719 -0
  23. package/dist/cheerio-compat-DcnYK9OB.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4120 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +35 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/native/vendor/scraper-0.24.0/Cargo.toml +52 -0
  167. package/native/vendor/scraper-0.24.0/LICENSE +15 -0
  168. package/native/vendor/scraper-0.24.0/src/element_ref/element.rs +212 -0
  169. package/native/vendor/scraper-0.24.0/src/element_ref/mod.rs +236 -0
  170. package/native/vendor/scraper-0.24.0/src/element_ref/serializable.rs +15 -0
  171. package/native/vendor/scraper-0.24.0/src/error/utils.rs +91 -0
  172. package/native/vendor/scraper-0.24.0/src/error.rs +118 -0
  173. package/native/vendor/scraper-0.24.0/src/html/mod.rs +270 -0
  174. package/native/vendor/scraper-0.24.0/src/html/serializable.rs +27 -0
  175. package/native/vendor/scraper-0.24.0/src/html/tree_sink.rs +285 -0
  176. package/native/vendor/scraper-0.24.0/src/lib.rs +180 -0
  177. package/native/vendor/scraper-0.24.0/src/node/serializable.rs +52 -0
  178. package/native/vendor/scraper-0.24.0/src/node.rs +465 -0
  179. package/native/vendor/scraper-0.24.0/src/selectable.rs +70 -0
  180. package/native/vendor/scraper-0.24.0/src/selector.rs +290 -0
  181. package/package.json +167 -4
  182. package/scripts/check-native.mjs +15 -0
  183. package/scripts/check-release.mjs +234 -0
  184. package/scripts/fix-declarations.mjs +39 -0
  185. package/scripts/generate-notices.mjs +194 -0
  186. package/scripts/notices/README.md +13 -0
  187. package/scripts/notices/cheerio-MIT.txt +21 -0
  188. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  189. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  190. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  191. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  192. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  193. package/scripts/notices/napi-rs-MIT.txt +43 -0
  194. package/scripts/notices/neon-MIT.txt +19 -0
  195. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  196. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  197. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  198. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  199. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  200. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  201. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  202. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  203. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  204. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  205. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  206. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  207. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  208. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  209. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  210. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  211. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  212. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  213. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  214. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  215. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  216. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  217. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  218. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  219. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  220. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  221. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  222. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  223. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  224. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  225. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  226. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  227. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  228. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  229. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  230. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  231. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  232. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  233. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  234. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  235. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  236. package/scripts/notices/runtime.json +1632 -0
  237. package/scripts/notices/sources.json +561 -0
  238. package/scripts/notices/xray-MIT.txt +21 -0
  239. package/src/attribute-object.ts +71 -0
  240. package/src/attributes.ts +308 -0
  241. package/src/cheerio-compat/css.ts +58 -0
  242. package/src/cheerio-compat/render.ts +106 -0
  243. package/src/cheerio-compat/selectors.ts +51 -0
  244. package/src/cheerio-compat/types.ts +52 -0
  245. package/src/cheerio-compat.ts +2055 -0
  246. package/src/codec.ts +96 -0
  247. package/src/dom.ts +87 -0
  248. package/src/extract-plan.ts +97 -0
  249. package/src/extract.ts +322 -0
  250. package/src/hooks.ts +369 -0
  251. package/src/index.ts +48 -0
  252. package/src/io.ts +491 -0
  253. package/src/match.ts +207 -0
  254. package/src/modern.ts +259 -0
  255. package/src/native.ts +149 -0
  256. package/src/nodes.ts +831 -0
  257. package/src/parse.ts +53 -0
  258. package/src/parser-types.ts +157 -0
  259. package/src/router.ts +1129 -0
  260. package/src/selection.ts +682 -0
  261. package/src/slim.ts +12 -0
  262. package/src/snapshot.ts +291 -0
  263. package/src/transport.ts +113 -0
  264. package/src/types.ts +186 -0
  265. package/src/utils.ts +29 -0
  266. package/src/workers.ts +21 -0
  267. package/tsconfig.json +15 -0
  268. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,2628 @@
1
+ // Copyright 2014-2017 The html5ever Project Developers. See the
2
+ // COPYRIGHT file at the top-level directory of this distribution.
3
+ //
4
+ // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5
+ // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6
+ // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7
+ // option. This file may not be copied, modified, or distributed
8
+ // except according to those terms.
9
+
10
+ //! The HTML5 tokenizer.
11
+
12
+ pub use self::interface::{CharacterTokens, EOFToken, NullCharacterToken, ParseError};
13
+ pub use self::interface::{CommentToken, DoctypeToken, TagToken, Token};
14
+ pub use self::interface::{Doctype, EndTag, StartTag, Tag, TagKind};
15
+ pub use self::interface::{TokenSink, TokenSinkResult};
16
+
17
+ use self::states::{DoctypeIdKind, Public, System};
18
+ use self::states::{DoubleEscaped, Escaped};
19
+ use self::states::{DoubleQuoted, SingleQuoted, Unquoted};
20
+ use self::states::{Rawtext, Rcdata, ScriptData, ScriptDataEscaped};
21
+
22
+ use self::char_ref::{CharRef, CharRefTokenizer};
23
+
24
+ use crate::util::str::lower_ascii_letter;
25
+
26
+ use log::{debug, trace};
27
+ use markup5ever::{ns, small_char_set, TokenizerResult};
28
+ use std::borrow::Cow::{self, Borrowed};
29
+ use std::cell::{Cell, RefCell, RefMut};
30
+ use std::collections::BTreeMap;
31
+ use std::mem;
32
+
33
+ pub use crate::buffer_queue::{BufferQueue, FromSet, NotFromSet, SetResult};
34
+ use crate::macros::{time, unwrap_or_return};
35
+ use crate::tendril::StrTendril;
36
+ use crate::{Attribute, LocalName, QualName, SmallCharSet};
37
+
38
+ mod char_ref;
39
+ mod interface;
40
+ pub mod states;
41
+
42
+ pub enum ProcessResult<Handle> {
43
+ Continue,
44
+ Suspend,
45
+ Script(Handle),
46
+ }
47
+
48
+ fn option_push(opt_str: &mut Option<StrTendril>, c: char) {
49
+ match *opt_str {
50
+ Some(ref mut s) => s.push_char(c),
51
+ None => *opt_str = Some(StrTendril::from_char(c)),
52
+ }
53
+ }
54
+
55
+ /// Tokenizer options, with an impl for `Default`.
56
+ #[derive(Clone)]
57
+ pub struct TokenizerOpts {
58
+ /// Report all parse errors described in the spec, at some
59
+ /// performance penalty? Default: false
60
+ pub exact_errors: bool,
61
+
62
+ /// Escape decoded private-use characters for cheerox's lossless UTF-16
63
+ /// transport. This extension is opt-in; ordinary html5ever users retain
64
+ /// the upstream character-reference behavior.
65
+ pub cheerox_codec: bool,
66
+
67
+ /// Discard a `U+FEFF BYTE ORDER MARK` if we see one at the beginning
68
+ /// of the stream? Default: true
69
+ pub discard_bom: bool,
70
+
71
+ /// Keep a record of how long we spent in each state? Printed
72
+ /// when `end()` is called. Default: false
73
+ pub profile: bool,
74
+
75
+ /// Initial state override. Only the test runner should use
76
+ /// a non-`None` value!
77
+ pub initial_state: Option<states::State>,
78
+
79
+ /// Last start tag. Only the test runner should use a
80
+ /// non-`None` value!
81
+ ///
82
+ /// FIXME: Can't use Tendril because we want TokenizerOpts
83
+ /// to be Send.
84
+ pub last_start_tag_name: Option<String>,
85
+ }
86
+
87
+ impl Default for TokenizerOpts {
88
+ fn default() -> TokenizerOpts {
89
+ TokenizerOpts {
90
+ exact_errors: false,
91
+ cheerox_codec: false,
92
+ discard_bom: true,
93
+ profile: false,
94
+ initial_state: None,
95
+ last_start_tag_name: None,
96
+ }
97
+ }
98
+ }
99
+
100
+ /// The HTML tokenizer.
101
+ pub struct Tokenizer<Sink> {
102
+ /// Options controlling the behavior of the tokenizer.
103
+ opts: TokenizerOpts,
104
+
105
+ /// Destination for tokens we emit.
106
+ pub sink: Sink,
107
+
108
+ /// The abstract machine state as described in the spec.
109
+ state: Cell<states::State>,
110
+
111
+ /// Are we at the end of the file, once buffers have been processed
112
+ /// completely? This affects whether we will wait for lookahead or not.
113
+ at_eof: Cell<bool>,
114
+
115
+ /// Tokenizer for character references, if we're tokenizing
116
+ /// one at the moment.
117
+ char_ref_tokenizer: RefCell<Option<Box<CharRefTokenizer>>>,
118
+
119
+ /// Current input character. Just consumed, may reconsume.
120
+ current_char: Cell<char>,
121
+
122
+ /// Should we reconsume the current input character?
123
+ reconsume: Cell<bool>,
124
+
125
+ /// Did we just consume \r, translating it to \n? In that case we need
126
+ /// to ignore the next character if it's \n.
127
+ ignore_lf: Cell<bool>,
128
+
129
+ /// Discard a U+FEFF BYTE ORDER MARK if we see one? Only done at the
130
+ /// beginning of the stream.
131
+ discard_bom: Cell<bool>,
132
+
133
+ /// Current tag kind.
134
+ current_tag_kind: Cell<TagKind>,
135
+
136
+ /// Current tag name.
137
+ current_tag_name: RefCell<StrTendril>,
138
+
139
+ /// Current tag is self-closing?
140
+ current_tag_self_closing: Cell<bool>,
141
+
142
+ /// Current tag attributes.
143
+ current_tag_attrs: RefCell<Vec<Attribute>>,
144
+
145
+ /// Current attribute name.
146
+ current_attr_name: RefCell<StrTendril>,
147
+
148
+ /// Current attribute value.
149
+ current_attr_value: RefCell<StrTendril>,
150
+
151
+ /// Current comment.
152
+ current_comment: RefCell<StrTendril>,
153
+
154
+ /// Current doctype token.
155
+ current_doctype: RefCell<Doctype>,
156
+
157
+ /// Last start tag name, for use in checking "appropriate end tag".
158
+ last_start_tag_name: RefCell<Option<LocalName>>,
159
+
160
+ /// The "temporary buffer" mentioned in the spec.
161
+ temp_buf: RefCell<StrTendril>,
162
+
163
+ /// Record of how many ns we spent in each state, if profiling is enabled.
164
+ state_profile: RefCell<BTreeMap<states::State, u64>>,
165
+
166
+ /// Record of how many ns we spent in the token sink.
167
+ time_in_sink: Cell<u64>,
168
+
169
+ /// Track current line
170
+ current_line: Cell<u64>,
171
+
172
+ /// Reusable lowercase scratch for the whole-tag fast scan.
173
+ fast_scratch: RefCell<Vec<u8>>,
174
+
175
+ /// Fast-scan diagnostics: committed tags and bails to the machine.
176
+ fast_hits: Cell<u32>,
177
+ fast_bails: Cell<u32>,
178
+ }
179
+
180
+ impl<Sink: TokenSink> Tokenizer<Sink> {
181
+ /// Create a new tokenizer which feeds tokens to a particular `TokenSink`.
182
+ pub fn new(sink: Sink, mut opts: TokenizerOpts) -> Tokenizer<Sink> {
183
+ let start_tag_name = opts
184
+ .last_start_tag_name
185
+ .take()
186
+ .map(|s| LocalName::from(&*s));
187
+ let state = opts.initial_state.unwrap_or(states::Data);
188
+ let discard_bom = opts.discard_bom;
189
+ Tokenizer {
190
+ opts,
191
+ sink,
192
+ state: Cell::new(state),
193
+ char_ref_tokenizer: RefCell::new(None),
194
+ at_eof: Cell::new(false),
195
+ current_char: Cell::new('\0'),
196
+ reconsume: Cell::new(false),
197
+ ignore_lf: Cell::new(false),
198
+ discard_bom: Cell::new(discard_bom),
199
+ current_tag_kind: Cell::new(StartTag),
200
+ current_tag_name: RefCell::new(StrTendril::new()),
201
+ current_tag_self_closing: Cell::new(false),
202
+ current_tag_attrs: RefCell::new(vec![]),
203
+ current_attr_name: RefCell::new(StrTendril::new()),
204
+ current_attr_value: RefCell::new(StrTendril::new()),
205
+ current_comment: RefCell::new(StrTendril::new()),
206
+ current_doctype: RefCell::new(Doctype::default()),
207
+ last_start_tag_name: RefCell::new(start_tag_name),
208
+ temp_buf: RefCell::new(StrTendril::new()),
209
+ state_profile: RefCell::new(BTreeMap::new()),
210
+ time_in_sink: Cell::new(0),
211
+ current_line: Cell::new(1),
212
+ fast_scratch: RefCell::new(Vec::new()),
213
+ fast_hits: Cell::new(0),
214
+ fast_bails: Cell::new(0),
215
+ }
216
+ }
217
+
218
+ /// Whole-tag fast-scan counters (diagnostic).
219
+ pub fn fast_scan_stats(&self) -> (u32, u32) {
220
+ (self.fast_hits.get(), self.fast_bails.get())
221
+ }
222
+
223
+ #[inline]
224
+ fn fast_bail(&self, front: &[u8]) -> Option<ProcessResult<Sink::Handle>> {
225
+ let _ = front;
226
+ self.fast_bails.set(self.fast_bails.get() + 1);
227
+ None
228
+ }
229
+
230
+ /// Feed an input string into the tokenizer.
231
+ pub fn feed(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle> {
232
+ if input.is_empty() {
233
+ return TokenizerResult::Done;
234
+ }
235
+
236
+ if self.discard_bom.get() {
237
+ if let Some(c) = input.peek() {
238
+ if c == '\u{feff}' {
239
+ input.next();
240
+ }
241
+ } else {
242
+ return TokenizerResult::Done;
243
+ }
244
+ };
245
+
246
+ self.run(input)
247
+ }
248
+
249
+ pub fn set_plaintext_state(&self) {
250
+ self.state.set(states::Plaintext);
251
+ }
252
+
253
+ fn process_token(&self, token: Token) -> TokenSinkResult<Sink::Handle> {
254
+ if self.opts.profile {
255
+ let (ret, dt) = time!(self.sink.process_token(token, self.current_line.get()));
256
+ self.time_in_sink.set(self.time_in_sink.get() + dt);
257
+ ret
258
+ } else {
259
+ self.sink.process_token(token, self.current_line.get())
260
+ }
261
+ }
262
+
263
+ fn process_token_and_continue(&self, token: Token) {
264
+ assert!(matches!(
265
+ self.process_token(token),
266
+ TokenSinkResult::Continue
267
+ ));
268
+ }
269
+
270
+ //§ preprocessing-the-input-stream
271
+ // Get the next input character, which might be the character
272
+ // 'c' that we already consumed from the buffers.
273
+ fn get_preprocessed_char(&self, mut c: char, input: &BufferQueue) -> Option<char> {
274
+ if self.ignore_lf.get() {
275
+ self.ignore_lf.set(false);
276
+ if c == '\n' {
277
+ c = input.next()?;
278
+ }
279
+ }
280
+
281
+ if c == '\r' {
282
+ self.ignore_lf.set(true);
283
+ c = '\n';
284
+ }
285
+
286
+ if c == '\n' {
287
+ self.current_line.set(self.current_line.get() + 1);
288
+ }
289
+
290
+ if self.opts.exact_errors
291
+ && match c as u32 {
292
+ 0x01..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F..=0x9F | 0xFDD0..=0xFDEF => true,
293
+ n if (n & 0xFFFE) == 0xFFFE => true,
294
+ _ => false,
295
+ }
296
+ {
297
+ let msg = format!("Bad character {c}");
298
+ self.emit_error(Cow::Owned(msg));
299
+ }
300
+
301
+ trace!("got character {c}");
302
+ self.current_char.set(c);
303
+ Some(c)
304
+ }
305
+
306
+ //§ tokenization
307
+ // Get the next input character, if one is available.
308
+ fn get_char(&self, input: &BufferQueue) -> Option<char> {
309
+ if self.reconsume.get() {
310
+ self.reconsume.set(false);
311
+ Some(self.current_char.get())
312
+ } else {
313
+ input
314
+ .next()
315
+ .and_then(|c| self.get_preprocessed_char(c, input))
316
+ }
317
+ }
318
+
319
+ fn pop_except_from(&self, input: &BufferQueue, set: SmallCharSet) -> Option<SetResult> {
320
+ // Bail to the slow path for various corner cases.
321
+ // This means that `FromSet` can contain characters not in the set!
322
+ // It shouldn't matter because the fallback `FromSet` case should
323
+ // always do the same thing as the `NotFromSet` case.
324
+ if self.opts.exact_errors || self.reconsume.get() || self.ignore_lf.get() {
325
+ return self.get_char(input).map(FromSet);
326
+ }
327
+
328
+ let d = input.pop_except_from(set);
329
+ trace!("got characters {d:?}");
330
+ match d {
331
+ Some(FromSet(c)) => self.get_preprocessed_char(c, input).map(FromSet),
332
+
333
+ // NB: We don't set self.current_char for a run of characters not
334
+ // in the set. It shouldn't matter for the codepaths that use
335
+ // this.
336
+ _ => d,
337
+ }
338
+ }
339
+
340
+ // Check if the next characters are an ASCII case-insensitive match. See
341
+ // BufferQueue::eat.
342
+ //
343
+ // NB: this doesn't set the current input character.
344
+ fn eat(&self, input: &BufferQueue, pat: &str, eq: fn(&u8, &u8) -> bool) -> Option<bool> {
345
+ if self.ignore_lf.get() {
346
+ self.ignore_lf.set(false);
347
+ if self.peek(input) == Some('\n') {
348
+ self.discard_char(input);
349
+ }
350
+ }
351
+
352
+ input.push_front(mem::take(&mut self.temp_buf.borrow_mut()));
353
+ match input.eat(pat, eq) {
354
+ None if self.at_eof.get() => Some(false),
355
+ None => {
356
+ while let Some(data) = input.next() {
357
+ self.temp_buf.borrow_mut().push_char(data);
358
+ }
359
+ None
360
+ }
361
+ Some(matched) => Some(matched),
362
+ }
363
+ }
364
+
365
+ /// Run the state machine for as long as we can.
366
+ fn run(&self, input: &BufferQueue) -> TokenizerResult<Sink::Handle> {
367
+ if self.opts.profile {
368
+ loop {
369
+ let state = self.state.get();
370
+ let old_sink = self.time_in_sink.get();
371
+ let (run, mut dt) = time!(self.step(input));
372
+ dt -= (self.time_in_sink.get() - old_sink);
373
+ let new = match self.state_profile.borrow_mut().get_mut(&state) {
374
+ Some(x) => {
375
+ *x += dt;
376
+ false
377
+ }
378
+ None => true,
379
+ };
380
+ if new {
381
+ // do this here because of borrow shenanigans
382
+ self.state_profile.borrow_mut().insert(state, dt);
383
+ }
384
+ match run {
385
+ ProcessResult::Continue => (),
386
+ ProcessResult::Suspend => break,
387
+ ProcessResult::Script(node) => return TokenizerResult::Script(node),
388
+ }
389
+ }
390
+ } else {
391
+ loop {
392
+ match self.step(input) {
393
+ ProcessResult::Continue => (),
394
+ ProcessResult::Suspend => break,
395
+ ProcessResult::Script(node) => return TokenizerResult::Script(node),
396
+ }
397
+ }
398
+ }
399
+ TokenizerResult::Done
400
+ }
401
+
402
+ #[inline]
403
+ fn bad_char_error(&self) {
404
+ #[cfg(feature = "trace_tokenizer")]
405
+ trace!(" error");
406
+
407
+ let msg = if self.opts.exact_errors {
408
+ Cow::from("Bad character")
409
+ } else {
410
+ let c = self.current_char.get();
411
+ let state = self.state.get();
412
+ Cow::from(format!("Saw {c} in state {state:?}"))
413
+ };
414
+ self.emit_error(msg);
415
+ }
416
+
417
+ #[inline]
418
+ fn bad_eof_error(&self) {
419
+ #[cfg(feature = "trace_tokenizer")]
420
+ trace!(" error_eof");
421
+
422
+ let msg = if self.opts.exact_errors {
423
+ Cow::from("Unexpected EOF")
424
+ } else {
425
+ let state = self.state.get();
426
+ Cow::from(format!("Saw EOF in state {state:?}"))
427
+ };
428
+ self.emit_error(msg);
429
+ }
430
+
431
+ fn emit_char(&self, c: char) {
432
+ #[cfg(feature = "trace_tokenizer")]
433
+ trace!(" emit");
434
+
435
+ self.process_token_and_continue(match c {
436
+ '\0' => NullCharacterToken,
437
+ _ => CharacterTokens(StrTendril::from_char(c)),
438
+ });
439
+ }
440
+
441
+ // The string must not contain '\0'!
442
+ fn emit_chars(&self, b: StrTendril) {
443
+ self.process_token_and_continue(CharacterTokens(b));
444
+ }
445
+
446
+ /// Byte classes for the whole-tag fast scan. Every set is deliberately
447
+ /// smaller than what the spec allows: any other byte bails to the state
448
+ /// machine, which is fully spec-compliant.
449
+ #[inline]
450
+ fn fast_is_ascii_letter(b: u8) -> bool {
451
+ matches!(b, b'a'..=b'z' | b'A'..=b'Z')
452
+ }
453
+
454
+ #[inline]
455
+ fn fast_is_tag_name_byte(b: u8) -> bool {
456
+ matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9')
457
+ }
458
+
459
+ #[inline]
460
+ fn fast_is_attr_name_byte(b: u8) -> bool {
461
+ matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b':' | b'.' | b'@')
462
+ }
463
+
464
+ #[inline]
465
+ fn fast_is_ws(b: u8) -> bool {
466
+ matches!(b, b'\t' | b'\n' | 0x0C | b' ')
467
+ }
468
+
469
+ /// An attribute value's quoted byte: everything ASCII except the closing
470
+ /// quote, NUL, CR (newline normalization), `&` (char refs) and non-ASCII.
471
+ #[inline]
472
+ fn fast_is_quoted_value_byte(b: u8, quote: u8) -> bool {
473
+ b != quote && b != 0 && b != b'&' && b != b'\r' && b < 0x80
474
+ }
475
+
476
+ /// An unquoted value byte: printable ASCII except quote marks, `=`, `<`,
477
+ /// backtick and `&`; whitespace and `>` terminate it elsewhere.
478
+ #[inline]
479
+ fn fast_is_unquoted_value_byte(b: u8) -> bool {
480
+ matches!(b, 0x21..=0x7E)
481
+ && !matches!(b, b'"' | b'\'' | b'=' | b'<' | b'`' | b'&')
482
+ }
483
+
484
+ /// Scans a whole well-formed common tag from the front chunk and emits it
485
+ /// in one step, skipping the per-character state machine for the shapes
486
+ /// `<name attr="value" ...>` and `</name>` that dominate real markup. A
487
+ /// leading `/` coming through the TagOpen state is folded in, so end tags
488
+ /// commit without the machine's EndTagOpen hop. Returns `None` without
489
+ /// consuming anything whenever anything unusual appears; the state
490
+ /// machine then handles the byte exactly as before. Parse errors the
491
+ /// machine would emit are dropped: the document sink never surfaces
492
+ /// tokenizer errors.
493
+ fn try_fast_tag(
494
+ &self,
495
+ input: &BufferQueue,
496
+ kind: TagKind,
497
+ ) -> Option<ProcessResult<Sink::Handle>> {
498
+ if self.opts.exact_errors || self.reconsume.get() || self.ignore_lf.get() {
499
+ // Legitimate precondition, not a scan failure.
500
+ return None;
501
+ }
502
+ let mut front = input.peek_front_chunk_mut()?;
503
+ let bytes: &[u8] = front.as_bytes();
504
+ // A leading `/` comes through the TagOpen state: switch to an end tag
505
+ // and scan from the next byte, skipping the machine's EndTagOpen hop.
506
+ let mut kind = kind;
507
+ let offset = usize::from(bytes.first() == Some(&b'/') && kind == TagKind::StartTag);
508
+ if offset == 1 {
509
+ kind = TagKind::EndTag;
510
+ }
511
+ let view: &[u8] = if offset == 1 { &bytes[1..] } else { bytes };
512
+ let len = view.len();
513
+ let mut i;
514
+ let mut newlines = 0u64;
515
+
516
+ // First byte: a tag name must start with an ASCII letter here. The
517
+ // machine's TagOpen/EndTagOpen arms handle `!`, `?` and errors.
518
+ if !Self::fast_is_ascii_letter(*view.first()?) {
519
+ return self.fast_bail(bytes);
520
+ }
521
+ i = 1;
522
+ while i < len && Self::fast_is_tag_name_byte(view[i]) {
523
+ i += 1;
524
+ }
525
+ let name_range = (0, i);
526
+ let mut self_closing = false;
527
+
528
+ // Collect attributes as (name, value) byte ranges first; the commit
529
+ // below only runs when the whole tag parsed, so nothing is mutated
530
+ // on the bail path.
531
+ let mut attr_ranges: Vec<((usize, usize), (usize, usize))> = Vec::new();
532
+
533
+ // Dispatch on the byte after the name.
534
+ match view.get(i) {
535
+ Some(b'>') => i += 1,
536
+ Some(&b) if Self::fast_is_ws(b) => {
537
+ 'attrs: loop {
538
+ while i < len && Self::fast_is_ws(view[i]) {
539
+ if view[i] == b'\n' {
540
+ newlines += 1;
541
+ }
542
+ i += 1;
543
+ }
544
+ match view.get(i) {
545
+ None => return self.fast_bail(bytes),
546
+ Some(b'>') => {
547
+ i += 1;
548
+ break 'attrs;
549
+ }
550
+ Some(b'/') => match view.get(i + 1) {
551
+ Some(b'>') => {
552
+ i += 2;
553
+ self_closing = true;
554
+ break 'attrs;
555
+ }
556
+ _ => return self.fast_bail(bytes),
557
+ },
558
+ Some(_) => {
559
+ // Attribute name.
560
+ if view[i] == b'=' {
561
+ return self.fast_bail(bytes);
562
+ }
563
+ let name_start = i;
564
+ while i < len && Self::fast_is_attr_name_byte(view[i]) {
565
+ i += 1;
566
+ }
567
+ if i == name_start {
568
+ return self.fast_bail(bytes);
569
+ }
570
+ let name = (name_start, i);
571
+ match view.get(i) {
572
+ // After the name: whitespace, then a new
573
+ // attribute, self-closing or the end.
574
+ Some(&b) if Self::fast_is_ws(b) => {
575
+ attr_ranges.push((name, (i, i)));
576
+ continue 'attrs;
577
+ }
578
+ Some(b'>') => {
579
+ attr_ranges.push((name, (i, i)));
580
+ i += 1;
581
+ break 'attrs;
582
+ }
583
+ Some(b'=') => {
584
+ i += 1;
585
+ // BeforeAttributeValue.
586
+ while i < len && Self::fast_is_ws(view[i]) {
587
+ if view[i] == b'\n' {
588
+ newlines += 1;
589
+ }
590
+ i += 1;
591
+ }
592
+ let value = match view.get(i) {
593
+ None => return self.fast_bail(bytes),
594
+ Some(&b'"') => {
595
+ i += 1;
596
+ let start = i;
597
+ while i < len
598
+ && Self::fast_is_quoted_value_byte(view[i], b'"')
599
+ {
600
+ if view[i] == b'\n' {
601
+ newlines += 1;
602
+ }
603
+ i += 1;
604
+ }
605
+ if view.get(i) != Some(&b'"') {
606
+ return self.fast_bail(bytes);
607
+ }
608
+ i += 1;
609
+ (start, i - 1)
610
+ }
611
+ Some(&b'\'') => {
612
+ i += 1;
613
+ let start = i;
614
+ while i < len
615
+ && Self::fast_is_quoted_value_byte(view[i], b'\'')
616
+ {
617
+ if view[i] == b'\n' {
618
+ newlines += 1;
619
+ }
620
+ i += 1;
621
+ }
622
+ if view.get(i) != Some(&b'\'') {
623
+ return self.fast_bail(bytes);
624
+ }
625
+ i += 1;
626
+ (start, i - 1)
627
+ }
628
+ // `<div foo=>` keeps an empty value.
629
+ Some(b'>') => (i, i),
630
+ Some(_) => {
631
+ let start = i;
632
+ while i < len
633
+ && Self::fast_is_unquoted_value_byte(view[i])
634
+ {
635
+ i += 1;
636
+ }
637
+ if i == start || view.get(i).is_none() {
638
+ return self.fast_bail(bytes);
639
+ }
640
+ match view.get(i) {
641
+ Some(&b) if Self::fast_is_ws(b) => {}
642
+ Some(b'>') => {}
643
+ _ => return self.fast_bail(bytes),
644
+ }
645
+ (start, i)
646
+ }
647
+ };
648
+ attr_ranges.push((name, value));
649
+ continue 'attrs;
650
+ }
651
+ Some(b'/') => match view.get(i + 1) {
652
+ Some(b'>') => {
653
+ attr_ranges.push((name, (i, i)));
654
+ i += 2;
655
+ self_closing = true;
656
+ break 'attrs;
657
+ }
658
+ _ => return self.fast_bail(bytes),
659
+ },
660
+ _ => return self.fast_bail(bytes),
661
+ }
662
+ }
663
+ }
664
+ }
665
+ }
666
+ Some(b'/') => match view.get(i + 1) {
667
+ Some(b'>') => {
668
+ i += 2;
669
+ self_closing = true;
670
+ }
671
+ _ => return self.fast_bail(bytes),
672
+ },
673
+ _ => return self.fast_bail(bytes),
674
+ }
675
+
676
+ // Commit: everything below only runs for a fully parsed tag.
677
+ {
678
+ #[inline]
679
+ fn lowered<'a>(view: &[u8], range: (usize, usize), out: &'a mut Vec<u8>) -> &'a str {
680
+ out.clear();
681
+ out.extend(view[range.0..range.1].iter().map(|b| b.to_ascii_lowercase()));
682
+ std::str::from_utf8(out).unwrap_or("")
683
+ }
684
+ let scratch = &mut self.fast_scratch.borrow_mut();
685
+
686
+ *self.current_tag_name.borrow_mut() =
687
+ StrTendril::from_slice(lowered(view, name_range, scratch));
688
+
689
+ let mut attrs = std::mem::take(&mut *self.current_tag_attrs.borrow_mut());
690
+ attrs.clear();
691
+ for &(name, value) in &attr_ranges {
692
+ let name = lowered(view, name, scratch);
693
+ // Same first-wins duplicate handling as finish_attribute.
694
+ if attrs.iter().any(|a| &*a.name.local == name) {
695
+ continue;
696
+ }
697
+ attrs.push(Attribute {
698
+ name: QualName::new(None, ns!(), LocalName::from(name)),
699
+ value: StrTendril::from_slice(
700
+ std::str::from_utf8(&view[value.0..value.1]).unwrap_or(""),
701
+ ),
702
+ });
703
+ }
704
+ *self.current_tag_attrs.borrow_mut() = attrs;
705
+ }
706
+
707
+ self.current_tag_kind.set(kind);
708
+ self.current_tag_self_closing.set(self_closing);
709
+ self.current_line.set(self.current_line.get() + newlines);
710
+
711
+ // The machine reaches emit_current_tag with the post-tag state set
712
+ // (`emit_tag Data`); the sink result then overrides it for script,
713
+ // raw text and plaintext.
714
+ self.state.set(states::Data);
715
+
716
+ // Consume the scanned bytes from the front chunk.
717
+ let consumed = (i + offset) as u32;
718
+ let now_empty = consumed as usize >= bytes.len();
719
+ unsafe {
720
+ front.unsafe_pop_front(consumed);
721
+ }
722
+ drop(front);
723
+ if now_empty {
724
+ input.pop_front();
725
+ }
726
+ return Some(self.emit_current_tag());
727
+ }
728
+
729
+ fn emit_current_tag(&self) -> ProcessResult<Sink::Handle> {
730
+ self.finish_attribute();
731
+
732
+ let name = LocalName::from(&**self.current_tag_name.borrow());
733
+ self.current_tag_name.borrow_mut().clear();
734
+
735
+ match self.current_tag_kind.get() {
736
+ StartTag => {
737
+ *self.last_start_tag_name.borrow_mut() = Some(name.clone());
738
+ }
739
+ EndTag => {
740
+ if !self.current_tag_attrs.borrow().is_empty() {
741
+ self.emit_error(Borrowed("Attributes on an end tag"));
742
+ }
743
+ if self.current_tag_self_closing.get() {
744
+ self.emit_error(Borrowed("Self-closing end tag"));
745
+ }
746
+ }
747
+ }
748
+
749
+ let token = TagToken(Tag {
750
+ kind: self.current_tag_kind.get(),
751
+ name,
752
+ self_closing: self.current_tag_self_closing.get(),
753
+ attrs: std::mem::take(&mut self.current_tag_attrs.borrow_mut()),
754
+ });
755
+
756
+ match self.process_token(token) {
757
+ TokenSinkResult::Continue => ProcessResult::Continue,
758
+ TokenSinkResult::Plaintext => {
759
+ self.state.set(states::Plaintext);
760
+ ProcessResult::Continue
761
+ }
762
+ TokenSinkResult::Script(node) => {
763
+ self.state.set(states::Data);
764
+ ProcessResult::Script(node)
765
+ }
766
+ TokenSinkResult::RawData(kind) => {
767
+ self.state.set(states::RawData(kind));
768
+ ProcessResult::Continue
769
+ }
770
+ }
771
+ }
772
+
773
+ fn emit_temp_buf(&self) {
774
+ #[cfg(feature = "trace_tokenizer")]
775
+ trace!(" emit_temp");
776
+
777
+ // FIXME: Make sure that clearing on emit is spec-compatible.
778
+ let buf = mem::take(&mut *self.temp_buf.borrow_mut());
779
+ self.emit_chars(buf);
780
+ }
781
+
782
+ fn clear_temp_buf(&self) {
783
+ // Do this without a new allocation.
784
+ self.temp_buf.borrow_mut().clear();
785
+ }
786
+
787
+ fn emit_current_comment(&self) {
788
+ let comment = mem::take(&mut *self.current_comment.borrow_mut());
789
+ self.process_token_and_continue(CommentToken(comment));
790
+ }
791
+
792
+ fn discard_tag(&self) {
793
+ self.current_tag_name.borrow_mut().clear();
794
+ self.current_tag_self_closing.set(false);
795
+ *self.current_tag_attrs.borrow_mut() = vec![];
796
+ }
797
+
798
+ fn create_tag(&self, kind: TagKind, c: char) {
799
+ self.discard_tag();
800
+ self.current_tag_name.borrow_mut().push_char(c);
801
+ self.current_tag_kind.set(kind);
802
+ }
803
+
804
+ fn have_appropriate_end_tag(&self) -> bool {
805
+ match self.last_start_tag_name.borrow().as_ref() {
806
+ Some(last) => {
807
+ (self.current_tag_kind.get() == EndTag)
808
+ && (**self.current_tag_name.borrow() == **last)
809
+ }
810
+ None => false,
811
+ }
812
+ }
813
+
814
+ fn create_attribute(&self, c: char) {
815
+ self.finish_attribute();
816
+
817
+ self.current_attr_name.borrow_mut().push_char(c);
818
+ }
819
+
820
+ fn finish_attribute(&self) {
821
+ if self.current_attr_name.borrow().is_empty() {
822
+ return;
823
+ }
824
+
825
+ // Check for a duplicate attribute.
826
+ // FIXME: the spec says we should error as soon as the name is finished.
827
+ let dup = {
828
+ let name = &*self.current_attr_name.borrow();
829
+ self.current_tag_attrs
830
+ .borrow()
831
+ .iter()
832
+ .any(|a| *a.name.local == **name)
833
+ };
834
+
835
+ if dup {
836
+ self.emit_error(Borrowed("Duplicate attribute"));
837
+ self.current_attr_name.borrow_mut().clear();
838
+ self.current_attr_value.borrow_mut().clear();
839
+ } else {
840
+ let name = LocalName::from(&**self.current_attr_name.borrow());
841
+ self.current_attr_name.borrow_mut().clear();
842
+ self.current_tag_attrs.borrow_mut().push(Attribute {
843
+ // The tree builder will adjust the namespace if necessary.
844
+ // This only happens in foreign elements.
845
+ name: QualName::new(None, ns!(), name),
846
+ value: mem::take(&mut self.current_attr_value.borrow_mut()),
847
+ });
848
+ }
849
+ }
850
+
851
+ fn emit_current_doctype(&self) {
852
+ let doctype = self.current_doctype.take();
853
+ self.process_token_and_continue(DoctypeToken(doctype));
854
+ }
855
+
856
+ fn doctype_id(&self, kind: DoctypeIdKind) -> RefMut<'_, Option<StrTendril>> {
857
+ let current_doctype = self.current_doctype.borrow_mut();
858
+ match kind {
859
+ Public => RefMut::map(current_doctype, |d| &mut d.public_id),
860
+ System => RefMut::map(current_doctype, |d| &mut d.system_id),
861
+ }
862
+ }
863
+
864
+ fn clear_doctype_id(&self, kind: DoctypeIdKind) {
865
+ let mut id = self.doctype_id(kind);
866
+ match *id {
867
+ Some(ref mut s) => s.clear(),
868
+ None => *id = Some(StrTendril::new()),
869
+ }
870
+ }
871
+
872
+ fn consume_char_ref(&self) {
873
+ *self.char_ref_tokenizer.borrow_mut() = Some(Box::new(CharRefTokenizer::new(matches!(
874
+ self.state.get(),
875
+ states::AttributeValue(_)
876
+ ))));
877
+ }
878
+
879
+ fn emit_eof(&self) {
880
+ self.process_token_and_continue(EOFToken);
881
+ }
882
+
883
+ fn peek(&self, input: &BufferQueue) -> Option<char> {
884
+ if self.reconsume.get() {
885
+ Some(self.current_char.get())
886
+ } else {
887
+ input.peek()
888
+ }
889
+ }
890
+
891
+ fn discard_char(&self, input: &BufferQueue) {
892
+ // peek() deals in un-processed characters (no newline normalization), while get_char()
893
+ // does.
894
+ //
895
+ // since discard_char is supposed to be used in combination with peek(), discard_char must
896
+ // discard a single raw input character, not a normalized newline.
897
+ if self.reconsume.get() {
898
+ self.reconsume.set(false);
899
+ } else {
900
+ input.next();
901
+ }
902
+ }
903
+
904
+ fn emit_error(&self, error: Cow<'static, str>) {
905
+ self.process_token_and_continue(ParseError(error));
906
+ }
907
+ }
908
+ //§ END
909
+
910
+ // Shorthand for common state machine behaviors.
911
+ macro_rules! shorthand (
912
+ ( $me:ident : create_tag $kind:ident $c:expr ) => ( $me.create_tag($kind, $c) );
913
+ ( $me:ident : push_tag $c:expr ) => ( $me.current_tag_name.borrow_mut().push_char($c) );
914
+ ( $me:ident : discard_tag ) => ( $me.discard_tag() );
915
+ ( $me:ident : discard_char $input:expr ) => ( $me.discard_char($input) );
916
+ ( $me:ident : push_temp $c:expr ) => ( $me.temp_buf.borrow_mut().push_char($c) );
917
+ ( $me:ident : clear_temp ) => ( $me.clear_temp_buf() );
918
+ ( $me:ident : create_attr $c:expr ) => ( $me.create_attribute($c) );
919
+ ( $me:ident : push_name $c:expr ) => ( $me.current_attr_name.borrow_mut().push_char($c) );
920
+ ( $me:ident : push_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_char($c) );
921
+ ( $me:ident : append_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_tendril($c));
922
+ ( $me:ident : push_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_char($c) );
923
+ ( $me:ident : append_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_slice($c) );
924
+ ( $me:ident : emit_comment ) => ( $me.emit_current_comment() );
925
+ ( $me:ident : clear_comment ) => ( $me.current_comment.borrow_mut().clear() );
926
+ ( $me:ident : create_doctype ) => ( *$me.current_doctype.borrow_mut() = Doctype::default() );
927
+ ( $me:ident : push_doctype_name $c:expr ) => ( option_push(&mut $me.current_doctype.borrow_mut().name, $c) );
928
+ ( $me:ident : push_doctype_id $k:ident $c:expr ) => ( option_push(&mut $me.doctype_id($k), $c) );
929
+ ( $me:ident : clear_doctype_id $k:ident ) => ( $me.clear_doctype_id($k) );
930
+ ( $me:ident : force_quirks ) => ( $me.current_doctype.borrow_mut().force_quirks = true);
931
+ ( $me:ident : emit_doctype ) => ( $me.emit_current_doctype() );
932
+ );
933
+
934
+ // Tracing of tokenizer actions. This adds significant bloat and compile time,
935
+ // so it's behind a cfg flag.
936
+ #[cfg(feature = "trace_tokenizer")]
937
+ macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ({
938
+ trace!(" {:?}", stringify!($($cmds)*));
939
+ shorthand!($me : $($cmds)*);
940
+ }));
941
+
942
+ #[cfg(not(feature = "trace_tokenizer"))]
943
+ macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ( shorthand!($me: $($cmds)*) ) );
944
+
945
+ // A little DSL for sequencing shorthand actions.
946
+ macro_rules! go (
947
+ // A pattern like $($cmd:tt)* ; $($rest:tt)* causes parse ambiguity.
948
+ // We have to tell the parser how much lookahead we need.
949
+
950
+ ( $me:ident : $a:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a); go!($me: $($rest)*); });
951
+ ( $me:ident : $a:tt $b:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b); go!($me: $($rest)*); });
952
+ ( $me:ident : $a:tt $b:tt $c:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c); go!($me: $($rest)*); });
953
+ ( $me:ident : $a:tt $b:tt $c:tt $d:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c $d); go!($me: $($rest)*); });
954
+
955
+ // These can only come at the end.
956
+
957
+ ( $me:ident : to $s:ident ) => ({ $me.state.set(states::$s); return ProcessResult::Continue; });
958
+ ( $me:ident : to $s:ident $k1:expr ) => ({ $me.state.set(states::$s($k1)); return ProcessResult::Continue; });
959
+ ( $me:ident : to $s:ident $k1:ident $k2:expr ) => ({ $me.state.set(states::$s($k1($k2))); return ProcessResult::Continue; });
960
+
961
+ ( $me:ident : reconsume $s:ident ) => ({ $me.reconsume.set(true); go!($me: to $s); });
962
+ ( $me:ident : reconsume $s:ident $k1:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1); });
963
+ ( $me:ident : reconsume $s:ident $k1:ident $k2:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1 $k2); });
964
+
965
+ ( $me:ident : consume_char_ref ) => ({ $me.consume_char_ref(); return ProcessResult::Continue; });
966
+
967
+ // We have a default next state after emitting a tag, but the sink can override.
968
+ ( $me:ident : emit_tag $s:ident ) => ({
969
+ $me.state.set(states::$s);
970
+ return $me.emit_current_tag();
971
+ });
972
+
973
+ ( $me:ident : eof ) => ({ $me.emit_eof(); return ProcessResult::Suspend; });
974
+
975
+ // If nothing else matched, it's a single command
976
+ ( $me:ident : $($cmd:tt)+ ) => ( sh_trace!($me: $($cmd)+) );
977
+
978
+ // or nothing.
979
+ ( $me:ident : ) => (());
980
+ );
981
+
982
+ // This is a macro because it can cause early return
983
+ // from the function where it is used.
984
+ macro_rules! get_char ( ($me:expr, $input:expr) => (
985
+ unwrap_or_return!($me.get_char($input), ProcessResult::Suspend)
986
+ ));
987
+
988
+ macro_rules! peek ( ($me:expr, $input:expr) => (
989
+ unwrap_or_return!($me.peek($input), ProcessResult::Suspend)
990
+ ));
991
+
992
+ macro_rules! pop_except_from ( ($me:expr, $input:expr, $set:expr) => (
993
+ unwrap_or_return!($me.pop_except_from($input, $set), ProcessResult::Suspend)
994
+ ));
995
+
996
+ macro_rules! eat ( ($me:expr, $input:expr, $pat:expr) => (
997
+ unwrap_or_return!($me.eat($input, $pat, u8::eq_ignore_ascii_case), ProcessResult::Suspend)
998
+ ));
999
+
1000
+ macro_rules! eat_exact ( ($me:expr, $input:expr, $pat:expr) => (
1001
+ unwrap_or_return!($me.eat($input, $pat, u8::eq), ProcessResult::Suspend)
1002
+ ));
1003
+
1004
+ impl<Sink: TokenSink> Tokenizer<Sink> {
1005
+ // Run the state machine for a while.
1006
+ // Return true if we should be immediately re-invoked
1007
+ // (this just simplifies control flow vs. break / continue).
1008
+ #[allow(clippy::never_loop)]
1009
+ fn step(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle> {
1010
+ if self.char_ref_tokenizer.borrow().is_some() {
1011
+ return self.step_char_ref_tokenizer(input);
1012
+ }
1013
+
1014
+ trace!("processing in state {:?}", self.state);
1015
+ match self.state.get() {
1016
+ //§ data-state
1017
+ states::Data => loop {
1018
+ let set = small_char_set!('\r' '\0' '&' '<' '\n');
1019
+
1020
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
1021
+ let set_result = if !(self.opts.exact_errors
1022
+ || self.reconsume.get()
1023
+ || self.ignore_lf.get())
1024
+ && Self::is_supported_simd_feature_detected()
1025
+ {
1026
+ let front_buffer = input.peek_front_chunk_mut();
1027
+ let Some(mut front_buffer) = front_buffer else {
1028
+ return ProcessResult::Suspend;
1029
+ };
1030
+
1031
+ // Special case: The fast path is not worth taking if the first character is already in the set,
1032
+ // which is fairly common
1033
+ let first_char = front_buffer
1034
+ .chars()
1035
+ .next()
1036
+ .expect("Input buffers are never empty");
1037
+
1038
+ if matches!(first_char, '\r' | '\0' | '&' | '<' | '\n') {
1039
+ drop(front_buffer);
1040
+ self.pop_except_from(input, set)
1041
+ } else {
1042
+ // SAFETY:
1043
+ // This CPU is guaranteed to support SIMD due to the is_supported_simd_feature_detected check above
1044
+ let result = unsafe { self.data_state_simd_fast_path(&mut front_buffer) };
1045
+
1046
+ if front_buffer.is_empty() {
1047
+ drop(front_buffer);
1048
+ input.pop_front();
1049
+ }
1050
+
1051
+ result
1052
+ }
1053
+ } else {
1054
+ self.pop_except_from(input, set)
1055
+ };
1056
+
1057
+ #[cfg(not(any(
1058
+ target_arch = "x86",
1059
+ target_arch = "x86_64",
1060
+ target_arch = "aarch64"
1061
+ )))]
1062
+ let set_result = self.pop_except_from(input, set);
1063
+
1064
+ let Some(set_result) = set_result else {
1065
+ return ProcessResult::Suspend;
1066
+ };
1067
+ match set_result {
1068
+ FromSet('\0') => {
1069
+ self.bad_char_error();
1070
+ self.emit_char('\0');
1071
+ }
1072
+ FromSet('&') => go!(self: consume_char_ref),
1073
+ FromSet('<') => go!(self: to TagOpen),
1074
+ FromSet(c) => {
1075
+ self.emit_char(c);
1076
+ }
1077
+ NotFromSet(b) => self.emit_chars(b),
1078
+ }
1079
+ },
1080
+
1081
+ //§ rcdata-state
1082
+ states::RawData(Rcdata) => loop {
1083
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '&' '<' '\n')) {
1084
+ FromSet('\0') => {
1085
+ self.bad_char_error();
1086
+ self.emit_char('\u{fffd}');
1087
+ }
1088
+ FromSet('&') => go!(self: consume_char_ref),
1089
+ FromSet('<') => go!(self: to RawLessThanSign Rcdata),
1090
+ FromSet(c) => self.emit_char(c),
1091
+ NotFromSet(b) => self.emit_chars(b),
1092
+ }
1093
+ },
1094
+
1095
+ //§ rawtext-state
1096
+ states::RawData(Rawtext) => loop {
1097
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) {
1098
+ FromSet('\0') => {
1099
+ self.bad_char_error();
1100
+ self.emit_char('\u{fffd}');
1101
+ }
1102
+ FromSet('<') => go!(self: to RawLessThanSign Rawtext),
1103
+ FromSet(c) => self.emit_char(c),
1104
+ NotFromSet(b) => self.emit_chars(b),
1105
+ }
1106
+ },
1107
+
1108
+ //§ script-data-state
1109
+ states::RawData(ScriptData) => loop {
1110
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) {
1111
+ FromSet('\0') => {
1112
+ self.bad_char_error();
1113
+ self.emit_char('\u{fffd}');
1114
+ }
1115
+ FromSet('<') => go!(self: to RawLessThanSign ScriptData),
1116
+ FromSet(c) => self.emit_char(c),
1117
+ NotFromSet(b) => self.emit_chars(b),
1118
+ }
1119
+ },
1120
+
1121
+ //§ script-data-escaped-state
1122
+ states::RawData(ScriptDataEscaped(Escaped)) => loop {
1123
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) {
1124
+ FromSet('\0') => {
1125
+ self.bad_char_error();
1126
+ self.emit_char('\u{fffd}');
1127
+ }
1128
+ FromSet('-') => {
1129
+ self.emit_char('-');
1130
+ go!(self: to ScriptDataEscapedDash Escaped);
1131
+ }
1132
+ FromSet('<') => go!(self: to RawLessThanSign ScriptDataEscaped Escaped),
1133
+ FromSet(c) => self.emit_char(c),
1134
+ NotFromSet(b) => self.emit_chars(b),
1135
+ }
1136
+ },
1137
+
1138
+ //§ script-data-double-escaped-state
1139
+ states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop {
1140
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) {
1141
+ FromSet('\0') => {
1142
+ self.bad_char_error();
1143
+ self.emit_char('\u{fffd}');
1144
+ }
1145
+ FromSet('-') => {
1146
+ self.emit_char('-');
1147
+ go!(self: to ScriptDataEscapedDash DoubleEscaped);
1148
+ }
1149
+ FromSet('<') => {
1150
+ self.emit_char('<');
1151
+ go!(self: to RawLessThanSign ScriptDataEscaped DoubleEscaped)
1152
+ }
1153
+ FromSet(c) => self.emit_char(c),
1154
+ NotFromSet(b) => self.emit_chars(b),
1155
+ }
1156
+ },
1157
+
1158
+ //§ plaintext-state
1159
+ states::Plaintext => loop {
1160
+ match pop_except_from!(self, input, small_char_set!('\r' '\0' '\n')) {
1161
+ FromSet('\0') => {
1162
+ self.bad_char_error();
1163
+ self.emit_char('\u{fffd}');
1164
+ }
1165
+ FromSet(c) => self.emit_char(c),
1166
+ NotFromSet(b) => self.emit_chars(b),
1167
+ }
1168
+ },
1169
+
1170
+ //§ tag-open-state
1171
+ states::TagOpen => loop {
1172
+ if let Some(result) = self.try_fast_tag(input, StartTag) {
1173
+ self.fast_hits.set(self.fast_hits.get() + 1);
1174
+ return result;
1175
+ }
1176
+ match get_char!(self, input) {
1177
+ '!' => go!(self: to MarkupDeclarationOpen),
1178
+ '/' => go!(self: to EndTagOpen),
1179
+ '?' => {
1180
+ self.bad_char_error();
1181
+ go!(self: clear_comment; reconsume BogusComment)
1182
+ }
1183
+ c => match lower_ascii_letter(c) {
1184
+ Some(cl) => go!(self: create_tag StartTag cl; to TagName),
1185
+ None => {
1186
+ self.bad_char_error();
1187
+ self.emit_char('<');
1188
+ go!(self: reconsume Data)
1189
+ }
1190
+ },
1191
+ }
1192
+ },
1193
+
1194
+ //§ end-tag-open-state
1195
+ states::EndTagOpen => loop {
1196
+ if let Some(result) = self.try_fast_tag(input, EndTag) {
1197
+ self.fast_hits.set(self.fast_hits.get() + 1);
1198
+ return result;
1199
+ }
1200
+ match get_char!(self, input) {
1201
+ '>' => {
1202
+ self.bad_char_error();
1203
+ go!(self: to Data)
1204
+ }
1205
+ c => match lower_ascii_letter(c) {
1206
+ Some(cl) => go!(self: create_tag EndTag cl; to TagName),
1207
+ None => {
1208
+ self.bad_char_error();
1209
+ go!(self: clear_comment; reconsume BogusComment)
1210
+ }
1211
+ },
1212
+ }
1213
+ },
1214
+
1215
+ //§ tag-name-state
1216
+ states::TagName => loop {
1217
+ match get_char!(self, input) {
1218
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName),
1219
+ '/' => go!(self: to SelfClosingStartTag),
1220
+ '>' => go!(self: emit_tag Data),
1221
+ '\0' => {
1222
+ self.bad_char_error();
1223
+ go!(self: push_tag '\u{fffd}')
1224
+ }
1225
+ c => go!(self: push_tag (c.to_ascii_lowercase())),
1226
+ }
1227
+ },
1228
+
1229
+ //§ script-data-escaped-less-than-sign-state
1230
+ states::RawLessThanSign(ScriptDataEscaped(Escaped)) => loop {
1231
+ match get_char!(self, input) {
1232
+ '/' => go!(self: clear_temp; to RawEndTagOpen ScriptDataEscaped Escaped),
1233
+ c => match lower_ascii_letter(c) {
1234
+ Some(cl) => {
1235
+ go!(self: clear_temp; push_temp cl);
1236
+ self.emit_char('<');
1237
+ self.emit_char(c);
1238
+ go!(self: to ScriptDataEscapeStart DoubleEscaped);
1239
+ }
1240
+ None => {
1241
+ self.emit_char('<');
1242
+ go!(self: reconsume RawData ScriptDataEscaped Escaped);
1243
+ }
1244
+ },
1245
+ }
1246
+ },
1247
+
1248
+ //§ script-data-double-escaped-less-than-sign-state
1249
+ states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => loop {
1250
+ match get_char!(self, input) {
1251
+ '/' => {
1252
+ go!(self: clear_temp);
1253
+ self.emit_char('/');
1254
+ go!(self: to ScriptDataDoubleEscapeEnd);
1255
+ }
1256
+ _ => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped),
1257
+ }
1258
+ },
1259
+
1260
+ //§ rcdata-less-than-sign-state rawtext-less-than-sign-state script-data-less-than-sign-state
1261
+ // otherwise
1262
+ states::RawLessThanSign(kind) => loop {
1263
+ match get_char!(self, input) {
1264
+ '/' => go!(self: clear_temp; to RawEndTagOpen kind),
1265
+ '!' if kind == ScriptData => {
1266
+ self.emit_char('<');
1267
+ self.emit_char('!');
1268
+ go!(self: to ScriptDataEscapeStart Escaped);
1269
+ }
1270
+ _ => {
1271
+ self.emit_char('<');
1272
+ go!(self: reconsume RawData kind);
1273
+ }
1274
+ }
1275
+ },
1276
+
1277
+ //§ rcdata-end-tag-open-state rawtext-end-tag-open-state script-data-end-tag-open-state script-data-escaped-end-tag-open-state
1278
+ states::RawEndTagOpen(kind) => loop {
1279
+ let c = get_char!(self, input);
1280
+ match lower_ascii_letter(c) {
1281
+ Some(cl) => go!(self: create_tag EndTag cl; push_temp c; to RawEndTagName kind),
1282
+ None => {
1283
+ self.emit_char('<');
1284
+ self.emit_char('/');
1285
+ go!(self: reconsume RawData kind);
1286
+ }
1287
+ }
1288
+ },
1289
+
1290
+ //§ rcdata-end-tag-name-state rawtext-end-tag-name-state script-data-end-tag-name-state script-data-escaped-end-tag-name-state
1291
+ states::RawEndTagName(kind) => loop {
1292
+ let c = get_char!(self, input);
1293
+ if self.have_appropriate_end_tag() {
1294
+ match c {
1295
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to BeforeAttributeName),
1296
+ '/' => go!(self: clear_temp; to SelfClosingStartTag),
1297
+ '>' => go!(self: clear_temp; emit_tag Data),
1298
+ _ => (),
1299
+ }
1300
+ }
1301
+
1302
+ match lower_ascii_letter(c) {
1303
+ Some(cl) => go!(self: push_tag cl; push_temp c),
1304
+ None => {
1305
+ go!(self: discard_tag);
1306
+ self.emit_char('<');
1307
+ self.emit_char('/');
1308
+ self.emit_temp_buf();
1309
+ go!(self: reconsume RawData kind);
1310
+ }
1311
+ }
1312
+ },
1313
+
1314
+ //§ script-data-double-escape-start-state
1315
+ states::ScriptDataEscapeStart(DoubleEscaped) => loop {
1316
+ let c = get_char!(self, input);
1317
+ match c {
1318
+ '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => {
1319
+ let esc = if &**self.temp_buf.borrow() == "script" {
1320
+ DoubleEscaped
1321
+ } else {
1322
+ Escaped
1323
+ };
1324
+ self.emit_char(c);
1325
+ go!(self: to RawData ScriptDataEscaped esc);
1326
+ }
1327
+ _ => match lower_ascii_letter(c) {
1328
+ Some(cl) => {
1329
+ go!(self: push_temp cl);
1330
+ self.emit_char(c);
1331
+ }
1332
+ None => go!(self: reconsume RawData ScriptDataEscaped Escaped),
1333
+ },
1334
+ }
1335
+ },
1336
+
1337
+ //§ script-data-escape-start-state
1338
+ states::ScriptDataEscapeStart(Escaped) => loop {
1339
+ match get_char!(self, input) {
1340
+ '-' => {
1341
+ self.emit_char('-');
1342
+ go!(self: to ScriptDataEscapeStartDash);
1343
+ }
1344
+ _ => go!(self: reconsume RawData ScriptData),
1345
+ }
1346
+ },
1347
+
1348
+ //§ script-data-escape-start-dash-state
1349
+ states::ScriptDataEscapeStartDash => loop {
1350
+ match get_char!(self, input) {
1351
+ '-' => {
1352
+ self.emit_char('-');
1353
+ go!(self: to ScriptDataEscapedDashDash Escaped);
1354
+ }
1355
+ _ => go!(self: reconsume RawData ScriptData),
1356
+ }
1357
+ },
1358
+
1359
+ //§ script-data-escaped-dash-state script-data-double-escaped-dash-state
1360
+ states::ScriptDataEscapedDash(kind) => loop {
1361
+ match get_char!(self, input) {
1362
+ '-' => {
1363
+ self.emit_char('-');
1364
+ go!(self: to ScriptDataEscapedDashDash kind);
1365
+ }
1366
+ '<' => {
1367
+ if kind == DoubleEscaped {
1368
+ self.emit_char('<');
1369
+ }
1370
+ go!(self: to RawLessThanSign ScriptDataEscaped kind);
1371
+ }
1372
+ '\0' => {
1373
+ self.bad_char_error();
1374
+ self.emit_char('\u{fffd}');
1375
+ go!(self: to RawData ScriptDataEscaped kind)
1376
+ }
1377
+ c => {
1378
+ self.emit_char(c);
1379
+ go!(self: to RawData ScriptDataEscaped kind);
1380
+ }
1381
+ }
1382
+ },
1383
+
1384
+ //§ script-data-escaped-dash-dash-state script-data-double-escaped-dash-dash-state
1385
+ states::ScriptDataEscapedDashDash(kind) => loop {
1386
+ match get_char!(self, input) {
1387
+ '-' => {
1388
+ self.emit_char('-');
1389
+ }
1390
+ '<' => {
1391
+ if kind == DoubleEscaped {
1392
+ self.emit_char('<');
1393
+ }
1394
+ go!(self: to RawLessThanSign ScriptDataEscaped kind);
1395
+ }
1396
+ '>' => {
1397
+ self.emit_char('>');
1398
+ go!(self: to RawData ScriptData);
1399
+ }
1400
+ '\0' => {
1401
+ self.bad_char_error();
1402
+ self.emit_char('\u{fffd}');
1403
+ go!(self: to RawData ScriptDataEscaped kind)
1404
+ }
1405
+ c => {
1406
+ self.emit_char(c);
1407
+ go!(self: to RawData ScriptDataEscaped kind);
1408
+ }
1409
+ }
1410
+ },
1411
+
1412
+ //§ script-data-double-escape-end-state
1413
+ states::ScriptDataDoubleEscapeEnd => loop {
1414
+ let c = get_char!(self, input);
1415
+ match c {
1416
+ '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => {
1417
+ let esc = if &**self.temp_buf.borrow() == "script" {
1418
+ Escaped
1419
+ } else {
1420
+ DoubleEscaped
1421
+ };
1422
+ self.emit_char(c);
1423
+ go!(self: to RawData ScriptDataEscaped esc);
1424
+ }
1425
+ _ => match lower_ascii_letter(c) {
1426
+ Some(cl) => {
1427
+ go!(self: push_temp cl);
1428
+ self.emit_char(c);
1429
+ }
1430
+ None => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped),
1431
+ },
1432
+ }
1433
+ },
1434
+
1435
+ //§ before-attribute-name-state
1436
+ states::BeforeAttributeName => loop {
1437
+ match get_char!(self, input) {
1438
+ '\t' | '\n' | '\x0C' | ' ' => (),
1439
+ '/' => go!(self: to SelfClosingStartTag),
1440
+ '>' => go!(self: emit_tag Data),
1441
+ '\0' => {
1442
+ self.bad_char_error();
1443
+ go!(self: create_attr '\u{fffd}'; to AttributeName)
1444
+ }
1445
+ c => match lower_ascii_letter(c) {
1446
+ Some(cl) => go!(self: create_attr cl; to AttributeName),
1447
+ None => {
1448
+ if matches!(c, '"' | '\'' | '<' | '=') {
1449
+ self.bad_char_error();
1450
+ }
1451
+
1452
+ go!(self: create_attr c; to AttributeName);
1453
+ }
1454
+ },
1455
+ }
1456
+ },
1457
+
1458
+ //§ attribute-name-state
1459
+ states::AttributeName => loop {
1460
+ match get_char!(self, input) {
1461
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to AfterAttributeName),
1462
+ '/' => go!(self: to SelfClosingStartTag),
1463
+ '=' => go!(self: to BeforeAttributeValue),
1464
+ '>' => go!(self: emit_tag Data),
1465
+ '\0' => {
1466
+ self.bad_char_error();
1467
+ go!(self: push_name '\u{fffd}')
1468
+ }
1469
+ c => match lower_ascii_letter(c) {
1470
+ Some(cl) => go!(self: push_name cl),
1471
+ None => {
1472
+ if matches!(c, '"' | '\'' | '<') {
1473
+ self.bad_char_error();
1474
+ }
1475
+ go!(self: push_name c);
1476
+ }
1477
+ },
1478
+ }
1479
+ },
1480
+
1481
+ //§ after-attribute-name-state
1482
+ states::AfterAttributeName => loop {
1483
+ match get_char!(self, input) {
1484
+ '\t' | '\n' | '\x0C' | ' ' => (),
1485
+ '/' => go!(self: to SelfClosingStartTag),
1486
+ '=' => go!(self: to BeforeAttributeValue),
1487
+ '>' => go!(self: emit_tag Data),
1488
+ '\0' => {
1489
+ self.bad_char_error();
1490
+ go!(self: create_attr '\u{fffd}'; to AttributeName)
1491
+ }
1492
+ c => match lower_ascii_letter(c) {
1493
+ Some(cl) => go!(self: create_attr cl; to AttributeName),
1494
+ None => {
1495
+ if matches!(c, '"' | '\'' | '<') {
1496
+ self.bad_char_error();
1497
+ }
1498
+
1499
+ go!(self: create_attr c; to AttributeName);
1500
+ }
1501
+ },
1502
+ }
1503
+ },
1504
+
1505
+ //§ before-attribute-value-state
1506
+ // Use peek so we can handle the first attr character along with the rest,
1507
+ // hopefully in the same zero-copy buffer.
1508
+ states::BeforeAttributeValue => loop {
1509
+ match peek!(self, input) {
1510
+ '\t' | '\n' | '\r' | '\x0C' | ' ' => go!(self: discard_char input),
1511
+ '"' => go!(self: discard_char input; to AttributeValue DoubleQuoted),
1512
+ '\'' => go!(self: discard_char input; to AttributeValue SingleQuoted),
1513
+ '>' => {
1514
+ go!(self: discard_char input);
1515
+ self.bad_char_error();
1516
+ go!(self: emit_tag Data)
1517
+ }
1518
+ _ => go!(self: to AttributeValue Unquoted),
1519
+ }
1520
+ },
1521
+
1522
+ //§ attribute-value-(double-quoted)-state
1523
+ states::AttributeValue(DoubleQuoted) => loop {
1524
+ match pop_except_from!(self, input, small_char_set!('\r' '"' '&' '\0' '\n')) {
1525
+ FromSet('"') => go!(self: to AfterAttributeValueQuoted),
1526
+ FromSet('&') => go!(self: consume_char_ref),
1527
+ FromSet('\0') => {
1528
+ self.bad_char_error();
1529
+ go!(self: push_value '\u{fffd}')
1530
+ }
1531
+ FromSet(c) => go!(self: push_value c),
1532
+ NotFromSet(ref b) => go!(self: append_value b),
1533
+ }
1534
+ },
1535
+
1536
+ //§ attribute-value-(single-quoted)-state
1537
+ states::AttributeValue(SingleQuoted) => loop {
1538
+ match pop_except_from!(self, input, small_char_set!('\r' '\'' '&' '\0' '\n')) {
1539
+ FromSet('\'') => go!(self: to AfterAttributeValueQuoted),
1540
+ FromSet('&') => go!(self: consume_char_ref),
1541
+ FromSet('\0') => {
1542
+ self.bad_char_error();
1543
+ go!(self: push_value '\u{fffd}')
1544
+ }
1545
+ FromSet(c) => go!(self: push_value c),
1546
+ NotFromSet(ref b) => go!(self: append_value b),
1547
+ }
1548
+ },
1549
+
1550
+ //§ attribute-value-(unquoted)-state
1551
+ states::AttributeValue(Unquoted) => loop {
1552
+ match pop_except_from!(
1553
+ self,
1554
+ input,
1555
+ small_char_set!('\r' '\t' '\n' '\x0C' ' ' '&' '>' '\0')
1556
+ ) {
1557
+ FromSet('\t') | FromSet('\n') | FromSet('\x0C') | FromSet(' ') => {
1558
+ go!(self: to BeforeAttributeName)
1559
+ }
1560
+ FromSet('&') => go!(self: consume_char_ref),
1561
+ FromSet('>') => go!(self: emit_tag Data),
1562
+ FromSet('\0') => {
1563
+ self.bad_char_error();
1564
+ go!(self: push_value '\u{fffd}')
1565
+ }
1566
+ FromSet(c) => {
1567
+ if matches!(c, '"' | '\'' | '<' | '=' | '`') {
1568
+ self.bad_char_error();
1569
+ }
1570
+ go!(self: push_value c);
1571
+ }
1572
+ NotFromSet(ref b) => go!(self: append_value b),
1573
+ }
1574
+ },
1575
+
1576
+ //§ after-attribute-value-(quoted)-state
1577
+ states::AfterAttributeValueQuoted => loop {
1578
+ match get_char!(self, input) {
1579
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName),
1580
+ '/' => go!(self: to SelfClosingStartTag),
1581
+ '>' => go!(self: emit_tag Data),
1582
+ _ => {
1583
+ self.bad_char_error();
1584
+ go!(self: reconsume BeforeAttributeName)
1585
+ }
1586
+ }
1587
+ },
1588
+
1589
+ //§ self-closing-start-tag-state
1590
+ states::SelfClosingStartTag => loop {
1591
+ match get_char!(self, input) {
1592
+ '>' => {
1593
+ self.current_tag_self_closing.set(true);
1594
+ go!(self: emit_tag Data);
1595
+ }
1596
+ _ => {
1597
+ self.bad_char_error();
1598
+ go!(self: reconsume BeforeAttributeName)
1599
+ }
1600
+ }
1601
+ },
1602
+
1603
+ //§ comment-start-state
1604
+ states::CommentStart => loop {
1605
+ match get_char!(self, input) {
1606
+ '-' => go!(self: to CommentStartDash),
1607
+ '\0' => {
1608
+ self.bad_char_error();
1609
+ go!(self: push_comment '\u{fffd}'; to Comment)
1610
+ }
1611
+ '>' => {
1612
+ self.bad_char_error();
1613
+ go!(self: emit_comment; to Data)
1614
+ }
1615
+ c => go!(self: push_comment c; to Comment),
1616
+ }
1617
+ },
1618
+
1619
+ //§ comment-start-dash-state
1620
+ states::CommentStartDash => loop {
1621
+ match get_char!(self, input) {
1622
+ '-' => go!(self: to CommentEnd),
1623
+ '\0' => {
1624
+ self.bad_char_error();
1625
+ go!(self: append_comment "-\u{fffd}"; to Comment)
1626
+ }
1627
+ '>' => {
1628
+ self.bad_char_error();
1629
+ go!(self: emit_comment; to Data)
1630
+ }
1631
+ c => go!(self: push_comment '-'; push_comment c; to Comment),
1632
+ }
1633
+ },
1634
+
1635
+ //§ comment-state
1636
+ states::Comment => loop {
1637
+ match get_char!(self, input) {
1638
+ c @ '<' => go!(self: push_comment c; to CommentLessThanSign),
1639
+ '-' => go!(self: to CommentEndDash),
1640
+ '\0' => {
1641
+ self.bad_char_error();
1642
+ go!(self: push_comment '\u{fffd}')
1643
+ }
1644
+ c => go!(self: push_comment c),
1645
+ }
1646
+ },
1647
+
1648
+ //§ comment-less-than-sign-state
1649
+ states::CommentLessThanSign => loop {
1650
+ match get_char!(self, input) {
1651
+ c @ '!' => go!(self: push_comment c; to CommentLessThanSignBang),
1652
+ c @ '<' => go!(self: push_comment c),
1653
+ _ => go!(self: reconsume Comment),
1654
+ }
1655
+ },
1656
+
1657
+ //§ comment-less-than-sign-bang
1658
+ states::CommentLessThanSignBang => loop {
1659
+ match get_char!(self, input) {
1660
+ '-' => go!(self: to CommentLessThanSignBangDash),
1661
+ _ => go!(self: reconsume Comment),
1662
+ }
1663
+ },
1664
+
1665
+ //§ comment-less-than-sign-bang-dash
1666
+ states::CommentLessThanSignBangDash => loop {
1667
+ match get_char!(self, input) {
1668
+ '-' => go!(self: to CommentLessThanSignBangDashDash),
1669
+ _ => go!(self: reconsume CommentEndDash),
1670
+ }
1671
+ },
1672
+
1673
+ //§ comment-less-than-sign-bang-dash-dash
1674
+ states::CommentLessThanSignBangDashDash => loop {
1675
+ match get_char!(self, input) {
1676
+ '>' => go!(self: reconsume CommentEnd),
1677
+ _ => {
1678
+ self.bad_char_error();
1679
+ go!(self: reconsume CommentEnd)
1680
+ }
1681
+ }
1682
+ },
1683
+
1684
+ //§ comment-end-dash-state
1685
+ states::CommentEndDash => loop {
1686
+ match get_char!(self, input) {
1687
+ '-' => go!(self: to CommentEnd),
1688
+ '\0' => {
1689
+ self.bad_char_error();
1690
+ go!(self: append_comment "-\u{fffd}"; to Comment)
1691
+ }
1692
+ c => go!(self: push_comment '-'; push_comment c; to Comment),
1693
+ }
1694
+ },
1695
+
1696
+ //§ comment-end-state
1697
+ states::CommentEnd => loop {
1698
+ match get_char!(self, input) {
1699
+ '>' => go!(self: emit_comment; to Data),
1700
+ '!' => go!(self: to CommentEndBang),
1701
+ '-' => go!(self: push_comment '-'),
1702
+ _ => go!(self: append_comment "--"; reconsume Comment),
1703
+ }
1704
+ },
1705
+
1706
+ //§ comment-end-bang-state
1707
+ states::CommentEndBang => loop {
1708
+ match get_char!(self, input) {
1709
+ '-' => go!(self: append_comment "--!"; to CommentEndDash),
1710
+ '>' => {
1711
+ self.bad_char_error();
1712
+ go!(self: emit_comment; to Data)
1713
+ }
1714
+ '\0' => {
1715
+ self.bad_char_error();
1716
+ go!(self: append_comment "--!\u{fffd}"; to Comment)
1717
+ }
1718
+ c => go!(self: append_comment "--!"; push_comment c; to Comment),
1719
+ }
1720
+ },
1721
+
1722
+ //§ doctype-state
1723
+ states::Doctype => loop {
1724
+ match get_char!(self, input) {
1725
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeName),
1726
+ '>' => go!(self: reconsume BeforeDoctypeName),
1727
+ _ => {
1728
+ self.bad_char_error();
1729
+ go!(self: reconsume BeforeDoctypeName)
1730
+ }
1731
+ }
1732
+ },
1733
+
1734
+ //§ before-doctype-name-state
1735
+ states::BeforeDoctypeName => loop {
1736
+ match get_char!(self, input) {
1737
+ '\t' | '\n' | '\x0C' | ' ' => (),
1738
+ '\0' => {
1739
+ self.bad_char_error();
1740
+ go!(self: create_doctype; push_doctype_name '\u{fffd}'; to DoctypeName)
1741
+ }
1742
+ '>' => {
1743
+ self.bad_char_error();
1744
+ go!(self: create_doctype; force_quirks; emit_doctype; to Data)
1745
+ }
1746
+ c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase());
1747
+ to DoctypeName),
1748
+ }
1749
+ },
1750
+
1751
+ //§ doctype-name-state
1752
+ states::DoctypeName => loop {
1753
+ match get_char!(self, input) {
1754
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to AfterDoctypeName),
1755
+ '>' => go!(self: emit_doctype; to Data),
1756
+ '\0' => {
1757
+ self.bad_char_error();
1758
+ go!(self: push_doctype_name '\u{fffd}')
1759
+ }
1760
+ c => go!(self: push_doctype_name (c.to_ascii_lowercase())),
1761
+ }
1762
+ },
1763
+
1764
+ //§ after-doctype-name-state
1765
+ states::AfterDoctypeName => loop {
1766
+ if eat!(self, input, "public") {
1767
+ go!(self: to AfterDoctypeKeyword Public);
1768
+ } else if eat!(self, input, "system") {
1769
+ go!(self: to AfterDoctypeKeyword System);
1770
+ } else {
1771
+ match get_char!(self, input) {
1772
+ '\t' | '\n' | '\x0C' | ' ' => (),
1773
+ '>' => go!(self: emit_doctype; to Data),
1774
+ _ => {
1775
+ self.bad_char_error();
1776
+ go!(self: force_quirks; reconsume BogusDoctype)
1777
+ }
1778
+ }
1779
+ }
1780
+ },
1781
+
1782
+ //§ after-doctype-public-keyword-state after-doctype-system-keyword-state
1783
+ states::AfterDoctypeKeyword(kind) => loop {
1784
+ match get_char!(self, input) {
1785
+ '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeIdentifier kind),
1786
+ '"' => {
1787
+ self.bad_char_error();
1788
+ go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind)
1789
+ }
1790
+ '\'' => {
1791
+ self.bad_char_error();
1792
+ go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind)
1793
+ }
1794
+ '>' => {
1795
+ self.bad_char_error();
1796
+ go!(self: force_quirks; emit_doctype; to Data)
1797
+ }
1798
+ _ => {
1799
+ self.bad_char_error();
1800
+ go!(self: force_quirks; reconsume BogusDoctype)
1801
+ }
1802
+ }
1803
+ },
1804
+
1805
+ //§ before-doctype-public-identifier-state before-doctype-system-identifier-state
1806
+ states::BeforeDoctypeIdentifier(kind) => loop {
1807
+ match get_char!(self, input) {
1808
+ '\t' | '\n' | '\x0C' | ' ' => (),
1809
+ '"' => go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind),
1810
+ '\'' => go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind),
1811
+ '>' => {
1812
+ self.bad_char_error();
1813
+ go!(self: force_quirks; emit_doctype; to Data)
1814
+ }
1815
+ _ => {
1816
+ self.bad_char_error();
1817
+ go!(self: force_quirks; reconsume BogusDoctype)
1818
+ }
1819
+ }
1820
+ },
1821
+
1822
+ //§ doctype-public-identifier-(double-quoted)-state doctype-system-identifier-(double-quoted)-state
1823
+ states::DoctypeIdentifierDoubleQuoted(kind) => loop {
1824
+ match get_char!(self, input) {
1825
+ '"' => go!(self: to AfterDoctypeIdentifier kind),
1826
+ '\0' => {
1827
+ self.bad_char_error();
1828
+ go!(self: push_doctype_id kind '\u{fffd}')
1829
+ }
1830
+ '>' => {
1831
+ self.bad_char_error();
1832
+ go!(self: force_quirks; emit_doctype; to Data)
1833
+ }
1834
+ c => go!(self: push_doctype_id kind c),
1835
+ }
1836
+ },
1837
+
1838
+ //§ doctype-public-identifier-(single-quoted)-state doctype-system-identifier-(single-quoted)-state
1839
+ states::DoctypeIdentifierSingleQuoted(kind) => loop {
1840
+ match get_char!(self, input) {
1841
+ '\'' => go!(self: to AfterDoctypeIdentifier kind),
1842
+ '\0' => {
1843
+ self.bad_char_error();
1844
+ go!(self: push_doctype_id kind '\u{fffd}')
1845
+ }
1846
+ '>' => {
1847
+ self.bad_char_error();
1848
+ go!(self: force_quirks; emit_doctype; to Data)
1849
+ }
1850
+ c => go!(self: push_doctype_id kind c),
1851
+ }
1852
+ },
1853
+
1854
+ //§ after-doctype-public-identifier-state
1855
+ states::AfterDoctypeIdentifier(Public) => loop {
1856
+ match get_char!(self, input) {
1857
+ '\t' | '\n' | '\x0C' | ' ' => {
1858
+ go!(self: to BetweenDoctypePublicAndSystemIdentifiers)
1859
+ }
1860
+ '>' => go!(self: emit_doctype; to Data),
1861
+ '"' => {
1862
+ self.bad_char_error();
1863
+ go!(self: clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System)
1864
+ }
1865
+ '\'' => {
1866
+ self.bad_char_error();
1867
+ go!(self: clear_doctype_id System; to DoctypeIdentifierSingleQuoted System)
1868
+ }
1869
+ _ => {
1870
+ self.bad_char_error();
1871
+ go!(self: force_quirks; reconsume BogusDoctype)
1872
+ }
1873
+ }
1874
+ },
1875
+
1876
+ //§ after-doctype-system-identifier-state
1877
+ states::AfterDoctypeIdentifier(System) => loop {
1878
+ match get_char!(self, input) {
1879
+ '\t' | '\n' | '\x0C' | ' ' => (),
1880
+ '>' => go!(self: emit_doctype; to Data),
1881
+ _ => {
1882
+ self.bad_char_error();
1883
+ go!(self: reconsume BogusDoctype)
1884
+ }
1885
+ }
1886
+ },
1887
+
1888
+ //§ between-doctype-public-and-system-identifiers-state
1889
+ states::BetweenDoctypePublicAndSystemIdentifiers => loop {
1890
+ match get_char!(self, input) {
1891
+ '\t' | '\n' | '\x0C' | ' ' => (),
1892
+ '>' => go!(self: emit_doctype; to Data),
1893
+ '"' => {
1894
+ go!(self: clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System)
1895
+ }
1896
+ '\'' => {
1897
+ go!(self: clear_doctype_id System; to DoctypeIdentifierSingleQuoted System)
1898
+ }
1899
+ _ => {
1900
+ self.bad_char_error();
1901
+ go!(self: force_quirks; reconsume BogusDoctype)
1902
+ }
1903
+ }
1904
+ },
1905
+
1906
+ //§ bogus-doctype-state
1907
+ states::BogusDoctype => loop {
1908
+ match get_char!(self, input) {
1909
+ '>' => go!(self: emit_doctype; to Data),
1910
+ '\0' => {
1911
+ self.bad_char_error();
1912
+ }
1913
+ _ => (),
1914
+ }
1915
+ },
1916
+
1917
+ //§ bogus-comment-state
1918
+ states::BogusComment => loop {
1919
+ match get_char!(self, input) {
1920
+ '>' => go!(self: emit_comment; to Data),
1921
+ '\0' => {
1922
+ self.bad_char_error();
1923
+ go!(self: push_comment '\u{fffd}')
1924
+ }
1925
+ c => go!(self: push_comment c),
1926
+ }
1927
+ },
1928
+
1929
+ //§ markup-declaration-open-state
1930
+ states::MarkupDeclarationOpen => loop {
1931
+ if eat_exact!(self, input, "--") {
1932
+ go!(self: clear_comment; to CommentStart);
1933
+ } else if eat!(self, input, "doctype") {
1934
+ go!(self: to Doctype);
1935
+ } else {
1936
+ if self
1937
+ .sink
1938
+ .adjusted_current_node_present_but_not_in_html_namespace()
1939
+ && eat_exact!(self, input, "[CDATA[")
1940
+ {
1941
+ go!(self: clear_temp; to CdataSection);
1942
+ }
1943
+ self.bad_char_error();
1944
+ go!(self: clear_comment; to BogusComment);
1945
+ }
1946
+ },
1947
+
1948
+ //§ cdata-section-state
1949
+ states::CdataSection => loop {
1950
+ match get_char!(self, input) {
1951
+ ']' => go!(self: to CdataSectionBracket),
1952
+ '\0' => {
1953
+ self.emit_temp_buf();
1954
+ self.emit_char('\0');
1955
+ }
1956
+ c => go!(self: push_temp c),
1957
+ }
1958
+ },
1959
+
1960
+ //§ cdata-section-bracket
1961
+ states::CdataSectionBracket => match get_char!(self, input) {
1962
+ ']' => go!(self: to CdataSectionEnd),
1963
+ _ => go!(self: push_temp ']'; reconsume CdataSection),
1964
+ },
1965
+
1966
+ //§ cdata-section-end
1967
+ states::CdataSectionEnd => loop {
1968
+ match get_char!(self, input) {
1969
+ ']' => go!(self: push_temp ']'),
1970
+ '>' => {
1971
+ self.emit_temp_buf();
1972
+ go!(self: to Data);
1973
+ }
1974
+ _ => go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection),
1975
+ }
1976
+ },
1977
+ //§ END
1978
+ }
1979
+ }
1980
+
1981
+ fn step_char_ref_tokenizer(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle> {
1982
+ // FIXME HACK: Take and replace the tokenizer so we don't
1983
+ // double-mut-borrow self. This is why it's boxed.
1984
+ let mut tok = self.char_ref_tokenizer.take().unwrap();
1985
+ let outcome = tok.step(self, input);
1986
+
1987
+ let progress = match outcome {
1988
+ char_ref::Done => {
1989
+ self.process_char_ref(tok.get_result());
1990
+ return ProcessResult::Continue;
1991
+ }
1992
+
1993
+ char_ref::Stuck => ProcessResult::Suspend,
1994
+ char_ref::Progress => ProcessResult::Continue,
1995
+ };
1996
+
1997
+ *self.char_ref_tokenizer.borrow_mut() = Some(tok);
1998
+ progress
1999
+ }
2000
+
2001
+ fn process_char_ref(&self, char_ref: CharRef) {
2002
+ let CharRef {
2003
+ mut chars,
2004
+ mut num_chars,
2005
+ } = char_ref;
2006
+
2007
+ if num_chars == 0 {
2008
+ chars[0] = '&';
2009
+ num_chars = 1;
2010
+ }
2011
+
2012
+ for i in 0..num_chars {
2013
+ let c = chars[i as usize];
2014
+ let is_escaped =
2015
+ self.opts.cheerox_codec && ('\u{e000}'..='\u{e800}').contains(&c);
2016
+ match self.state.get() {
2017
+ // The escape and its character form one text token, so a sink
2018
+ // that forwards each token never receives them separately.
2019
+ states::Data | states::RawData(states::Rcdata) if is_escaped => {
2020
+ let mut text = StrTendril::from_char('\u{e800}');
2021
+ text.push_char(c);
2022
+ self.emit_chars(text);
2023
+ },
2024
+ states::Data | states::RawData(states::Rcdata) => self.emit_char(c),
2025
+
2026
+ states::AttributeValue(_) => {
2027
+ if is_escaped {
2028
+ go!(self: push_value '\u{e800}');
2029
+ }
2030
+ go!(self: push_value c);
2031
+ },
2032
+
2033
+ _ => panic!(
2034
+ "state {:?} should not be reachable in process_char_ref",
2035
+ self.state.get()
2036
+ ),
2037
+ }
2038
+ }
2039
+ }
2040
+
2041
+ /// Indicate that we have reached the end of the input.
2042
+ pub fn end(&self) {
2043
+ // Handle EOF in the char ref sub-tokenizer, if there is one.
2044
+ // Do this first because it might un-consume stuff.
2045
+ let input = BufferQueue::default();
2046
+ match self.char_ref_tokenizer.take() {
2047
+ None => (),
2048
+ Some(mut tok) => {
2049
+ tok.end_of_file(self, &input);
2050
+ self.process_char_ref(tok.get_result());
2051
+ }
2052
+ }
2053
+
2054
+ // Process all remaining buffered input.
2055
+ // If we're waiting for lookahead, we're not gonna get it.
2056
+ self.at_eof.set(true);
2057
+ assert!(matches!(self.run(&input), TokenizerResult::Done));
2058
+ assert!(input.is_empty());
2059
+
2060
+ loop {
2061
+ match self.eof_step() {
2062
+ ProcessResult::Continue => (),
2063
+ ProcessResult::Suspend => break,
2064
+ ProcessResult::Script(_) => unreachable!(),
2065
+ }
2066
+ }
2067
+
2068
+ self.sink.end();
2069
+
2070
+ if self.opts.profile {
2071
+ self.dump_profile();
2072
+ }
2073
+ }
2074
+
2075
+ fn dump_profile(&self) {
2076
+ let mut results: Vec<(states::State, u64)> = self
2077
+ .state_profile
2078
+ .borrow()
2079
+ .iter()
2080
+ .map(|(s, t)| (*s, *t))
2081
+ .collect();
2082
+ results.sort_by(|&(_, x), &(_, y)| y.cmp(&x));
2083
+
2084
+ let total: u64 = results
2085
+ .iter()
2086
+ .map(|&(_, t)| t)
2087
+ .fold(0, ::std::ops::Add::add);
2088
+ println!("\nTokenizer profile, in nanoseconds");
2089
+ println!(
2090
+ "\n{:12} total in token sink",
2091
+ self.time_in_sink.get()
2092
+ );
2093
+ println!("\n{total:12} total in tokenizer");
2094
+
2095
+ for (k, v) in results.into_iter() {
2096
+ let pct = 100.0 * (v as f64) / (total as f64);
2097
+ println!("{v:12} {pct:4.1}% {k:?}");
2098
+ }
2099
+ }
2100
+
2101
+ fn eof_step(&self) -> ProcessResult<Sink::Handle> {
2102
+ debug!("processing EOF in state {:?}", self.state.get());
2103
+ match self.state.get() {
2104
+ states::Data
2105
+ | states::RawData(Rcdata)
2106
+ | states::RawData(Rawtext)
2107
+ | states::RawData(ScriptData)
2108
+ | states::Plaintext => go!(self: eof),
2109
+
2110
+ states::TagName
2111
+ | states::RawData(ScriptDataEscaped(_))
2112
+ | states::BeforeAttributeName
2113
+ | states::AttributeName
2114
+ | states::AfterAttributeName
2115
+ | states::AttributeValue(_)
2116
+ | states::AfterAttributeValueQuoted
2117
+ | states::SelfClosingStartTag
2118
+ | states::ScriptDataEscapedDash(_)
2119
+ | states::ScriptDataEscapedDashDash(_) => {
2120
+ self.bad_eof_error();
2121
+ go!(self: to Data)
2122
+ }
2123
+
2124
+ states::BeforeAttributeValue => go!(self: reconsume AttributeValue Unquoted),
2125
+
2126
+ states::TagOpen => {
2127
+ self.bad_eof_error();
2128
+ self.emit_char('<');
2129
+ go!(self: to Data);
2130
+ }
2131
+
2132
+ states::EndTagOpen => {
2133
+ self.bad_eof_error();
2134
+ self.emit_char('<');
2135
+ self.emit_char('/');
2136
+ go!(self: to Data);
2137
+ }
2138
+
2139
+ states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => {
2140
+ go!(self: to RawData ScriptDataEscaped DoubleEscaped)
2141
+ }
2142
+
2143
+ states::RawLessThanSign(kind) => {
2144
+ self.emit_char('<');
2145
+ go!(self: to RawData kind);
2146
+ }
2147
+
2148
+ states::RawEndTagOpen(kind) => {
2149
+ self.emit_char('<');
2150
+ self.emit_char('/');
2151
+ go!(self: to RawData kind);
2152
+ }
2153
+
2154
+ states::RawEndTagName(kind) => {
2155
+ self.emit_char('<');
2156
+ self.emit_char('/');
2157
+ self.emit_temp_buf();
2158
+ go!(self: to RawData kind)
2159
+ }
2160
+
2161
+ states::ScriptDataEscapeStart(kind) => go!(self: to RawData ScriptDataEscaped kind),
2162
+
2163
+ states::ScriptDataEscapeStartDash => go!(self: to RawData ScriptData),
2164
+
2165
+ states::ScriptDataDoubleEscapeEnd => {
2166
+ go!(self: to RawData ScriptDataEscaped DoubleEscaped)
2167
+ }
2168
+
2169
+ states::CommentStart
2170
+ | states::CommentStartDash
2171
+ | states::Comment
2172
+ | states::CommentEndDash
2173
+ | states::CommentEnd
2174
+ | states::CommentEndBang => {
2175
+ self.bad_eof_error();
2176
+ go!(self: emit_comment; to Data)
2177
+ }
2178
+
2179
+ states::CommentLessThanSign | states::CommentLessThanSignBang => {
2180
+ go!(self: reconsume Comment)
2181
+ }
2182
+
2183
+ states::CommentLessThanSignBangDash => go!(self: reconsume CommentEndDash),
2184
+
2185
+ states::CommentLessThanSignBangDashDash => go!(self: reconsume CommentEnd),
2186
+
2187
+ states::Doctype | states::BeforeDoctypeName => {
2188
+ self.bad_eof_error();
2189
+ go!(self: create_doctype; force_quirks; emit_doctype; to Data)
2190
+ }
2191
+
2192
+ states::DoctypeName
2193
+ | states::AfterDoctypeName
2194
+ | states::AfterDoctypeKeyword(_)
2195
+ | states::BeforeDoctypeIdentifier(_)
2196
+ | states::DoctypeIdentifierDoubleQuoted(_)
2197
+ | states::DoctypeIdentifierSingleQuoted(_)
2198
+ | states::AfterDoctypeIdentifier(_)
2199
+ | states::BetweenDoctypePublicAndSystemIdentifiers => {
2200
+ self.bad_eof_error();
2201
+ go!(self: force_quirks; emit_doctype; to Data)
2202
+ }
2203
+
2204
+ states::BogusDoctype => go!(self: emit_doctype; to Data),
2205
+
2206
+ states::BogusComment => go!(self: emit_comment; to Data),
2207
+
2208
+ states::MarkupDeclarationOpen => {
2209
+ self.bad_char_error();
2210
+ go!(self: to BogusComment)
2211
+ }
2212
+
2213
+ states::CdataSection => {
2214
+ self.emit_temp_buf();
2215
+ self.bad_eof_error();
2216
+ go!(self: to Data)
2217
+ }
2218
+
2219
+ states::CdataSectionBracket => go!(self: push_temp ']'; to CdataSection),
2220
+
2221
+ states::CdataSectionEnd => go!(self: push_temp ']'; push_temp ']'; to CdataSection),
2222
+ }
2223
+ }
2224
+
2225
+ /// Checks for supported SIMD feature, which is now either SSE2 for x86/x86_64 or NEON for aarch64.
2226
+ fn is_supported_simd_feature_detected() -> bool {
2227
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
2228
+ {
2229
+ is_x86_feature_detected!("sse2")
2230
+ }
2231
+
2232
+ #[cfg(target_arch = "aarch64")]
2233
+ {
2234
+ std::arch::is_aarch64_feature_detected!("neon")
2235
+ }
2236
+
2237
+ #[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
2238
+ false
2239
+ }
2240
+
2241
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
2242
+ /// Implements the [data state] with SIMD instructions.
2243
+ /// Calls SSE2- or NEON-specific function for chunks and processes any remaining bytes.
2244
+ ///
2245
+ /// The algorithm implemented is the naive SIMD approach described [here].
2246
+ ///
2247
+ /// ### SAFETY:
2248
+ /// Calling this function on a CPU that supports neither SSE2 nor NEON causes undefined behaviour.
2249
+ ///
2250
+ /// [data state]: https://html.spec.whatwg.org/#data-state
2251
+ /// [here]: https://lemire.me/blog/2024/06/08/scan-html-faster-with-simd-instructions-chrome-edition/
2252
+ unsafe fn data_state_simd_fast_path(&self, input: &mut StrTendril) -> Option<SetResult> {
2253
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
2254
+ let (mut i, mut n_newlines) = self.data_state_sse2_fast_path(input);
2255
+
2256
+ #[cfg(target_arch = "aarch64")]
2257
+ let (mut i, mut n_newlines) = self.data_state_neon_fast_path(input);
2258
+
2259
+ // Process any remaining bytes (less than STRIDE)
2260
+ while let Some(c) = input.as_bytes().get(i) {
2261
+ if matches!(*c, b'<' | b'&' | b'\r' | b'\0') {
2262
+ break;
2263
+ }
2264
+ if *c == b'\n' {
2265
+ n_newlines += 1;
2266
+ }
2267
+
2268
+ i += 1;
2269
+ }
2270
+
2271
+ let set_result = if i == 0 {
2272
+ let first_char = input.pop_front_char().unwrap();
2273
+ debug_assert!(matches!(first_char, '<' | '&' | '\r' | '\0'));
2274
+
2275
+ // FIXME: Passing a bogus input queue is only relevant when c is \n, which can never happen in this case.
2276
+ // Still, it would be nice to not have to do that.
2277
+ // The same is true for the unwrap call.
2278
+ let preprocessed_char = self
2279
+ .get_preprocessed_char(first_char, &BufferQueue::default())
2280
+ .unwrap();
2281
+ SetResult::FromSet(preprocessed_char)
2282
+ } else {
2283
+ debug_assert!(
2284
+ input.len() >= i,
2285
+ "Trying to remove {:?} bytes from a tendril that is only {:?} bytes long",
2286
+ i,
2287
+ input.len()
2288
+ );
2289
+ let consumed_chunk = input.unsafe_subtendril(0, i as u32);
2290
+ input.unsafe_pop_front(i as u32);
2291
+ SetResult::NotFromSet(consumed_chunk)
2292
+ };
2293
+
2294
+ self.current_line.set(self.current_line.get() + n_newlines);
2295
+
2296
+ Some(set_result)
2297
+ }
2298
+
2299
+ #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
2300
+ #[target_feature(enable = "sse2")]
2301
+ /// Implements the [data state] with SSE2 instructions for x86/x86_64.
2302
+ /// Returns a pair of the number of bytes processed and the number of newlines found.
2303
+ ///
2304
+ /// ### SAFETY:
2305
+ /// Calling this function on a CPU that does not support NEON causes undefined behaviour.
2306
+ ///
2307
+ /// [data state]: https://html.spec.whatwg.org/#data-state
2308
+ unsafe fn data_state_sse2_fast_path(&self, input: &mut StrTendril) -> (usize, u64) {
2309
+ #[cfg(target_arch = "x86")]
2310
+ use std::arch::x86::{
2311
+ __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128,
2312
+ _mm_set1_epi8,
2313
+ };
2314
+ #[cfg(target_arch = "x86_64")]
2315
+ use std::arch::x86_64::{
2316
+ __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128,
2317
+ _mm_set1_epi8,
2318
+ };
2319
+
2320
+ debug_assert!(!input.is_empty());
2321
+
2322
+ let quote_mask = _mm_set1_epi8('<' as i8);
2323
+ let escape_mask = _mm_set1_epi8('&' as i8);
2324
+ let carriage_return_mask = _mm_set1_epi8('\r' as i8);
2325
+ let zero_mask = _mm_set1_epi8('\0' as i8);
2326
+ let newline_mask = _mm_set1_epi8('\n' as i8);
2327
+
2328
+ let raw_bytes: &[u8] = input.as_bytes();
2329
+ let start = raw_bytes.as_ptr();
2330
+
2331
+ const STRIDE: usize = 16;
2332
+ let mut i = 0;
2333
+ let mut n_newlines = 0;
2334
+ while i + STRIDE <= raw_bytes.len() {
2335
+ // Load a 16 byte chunk from the input
2336
+ let data = _mm_loadu_si128(start.add(i) as *const __m128i);
2337
+
2338
+ // Compare the chunk against each mask
2339
+ let quotes = _mm_cmpeq_epi8(data, quote_mask);
2340
+ let escapes = _mm_cmpeq_epi8(data, escape_mask);
2341
+ let carriage_returns = _mm_cmpeq_epi8(data, carriage_return_mask);
2342
+ let zeros = _mm_cmpeq_epi8(data, zero_mask);
2343
+ let newlines = _mm_cmpeq_epi8(data, newline_mask);
2344
+
2345
+ // Combine all test results and create a bitmask from them.
2346
+ // Each bit in the mask will be 1 if the character at the bit position is in the set and 0 otherwise.
2347
+ let test_result = _mm_or_si128(
2348
+ _mm_or_si128(quotes, zeros),
2349
+ _mm_or_si128(escapes, carriage_returns),
2350
+ );
2351
+ let bitmask = _mm_movemask_epi8(test_result);
2352
+ let newline_mask = _mm_movemask_epi8(newlines);
2353
+
2354
+ if (bitmask != 0) {
2355
+ // We have reached one of the characters that cause the state machine to transition
2356
+ let position = if cfg!(target_endian = "little") {
2357
+ bitmask.trailing_zeros() as usize
2358
+ } else {
2359
+ bitmask.leading_zeros() as usize
2360
+ };
2361
+
2362
+ n_newlines += (newline_mask & ((1 << position) - 1)).count_ones() as u64;
2363
+ i += position;
2364
+ break;
2365
+ } else {
2366
+ n_newlines += newline_mask.count_ones() as u64;
2367
+ }
2368
+
2369
+ i += STRIDE;
2370
+ }
2371
+
2372
+ (i, n_newlines)
2373
+ }
2374
+
2375
+ #[cfg(target_arch = "aarch64")]
2376
+ #[target_feature(enable = "neon")]
2377
+ /// Implements the [data state] with NEON SIMD instructions for AArch64.
2378
+ /// Returns a pair of the number of bytes processed and the number of newlines found.
2379
+ ///
2380
+ /// ### SAFETY:
2381
+ /// Calling this function on a CPU that does not support NEON causes undefined behaviour.
2382
+ ///
2383
+ /// [data state]: https://html.spec.whatwg.org/#data-state
2384
+ unsafe fn data_state_neon_fast_path(&self, input: &mut StrTendril) -> (usize, u64) {
2385
+ use std::arch::aarch64::{vceqq_u8, vdupq_n_u8, vld1q_u8, vmaxvq_u8, vorrq_u8};
2386
+
2387
+ debug_assert!(!input.is_empty());
2388
+
2389
+ let quote_mask = vdupq_n_u8(b'<');
2390
+ let escape_mask = vdupq_n_u8(b'&');
2391
+ let carriage_return_mask = vdupq_n_u8(b'\r');
2392
+ let zero_mask = vdupq_n_u8(b'\0');
2393
+ let newline_mask = vdupq_n_u8(b'\n');
2394
+
2395
+ let raw_bytes: &[u8] = input.as_bytes();
2396
+ let start = raw_bytes.as_ptr();
2397
+
2398
+ const STRIDE: usize = 16;
2399
+ let mut i = 0;
2400
+ let mut n_newlines = 0;
2401
+ while i + STRIDE <= raw_bytes.len() {
2402
+ // Load a 16 byte chunk from the input
2403
+ let data = vld1q_u8(start.add(i));
2404
+
2405
+ // Compare the chunk against each mask
2406
+ let quotes = vceqq_u8(data, quote_mask);
2407
+ let escapes = vceqq_u8(data, escape_mask);
2408
+ let carriage_returns = vceqq_u8(data, carriage_return_mask);
2409
+ let zeros = vceqq_u8(data, zero_mask);
2410
+ let newlines = vceqq_u8(data, newline_mask);
2411
+
2412
+ // Combine all test results and create a bitmask from them.
2413
+ // Each bit in the mask will be 1 if the character at the bit position is in the set and 0 otherwise.
2414
+ let test_result =
2415
+ vorrq_u8(vorrq_u8(quotes, zeros), vorrq_u8(escapes, carriage_returns));
2416
+ let bitmask = vmaxvq_u8(test_result);
2417
+ let newline_mask = vmaxvq_u8(newlines);
2418
+ if bitmask != 0 {
2419
+ // We have reached one of the characters that cause the state machine to transition
2420
+ let chunk_bytes = std::slice::from_raw_parts(start.add(i), STRIDE);
2421
+ let position = chunk_bytes
2422
+ .iter()
2423
+ .position(|&b| matches!(b, b'<' | b'&' | b'\r' | b'\0'))
2424
+ .unwrap();
2425
+
2426
+ n_newlines += chunk_bytes[..position]
2427
+ .iter()
2428
+ .filter(|&&b| b == b'\n')
2429
+ .count() as u64;
2430
+
2431
+ i += position;
2432
+ break;
2433
+ } else if newline_mask != 0 {
2434
+ let chunk_bytes = std::slice::from_raw_parts(start.add(i), STRIDE);
2435
+ n_newlines += chunk_bytes.iter().filter(|&&b| b == b'\n').count() as u64;
2436
+ }
2437
+
2438
+ i += STRIDE;
2439
+ }
2440
+
2441
+ (i, n_newlines)
2442
+ }
2443
+ }
2444
+
2445
+ #[cfg(test)]
2446
+ #[allow(non_snake_case)]
2447
+ mod test {
2448
+ use super::option_push; // private items
2449
+ use crate::tendril::{SliceExt, StrTendril};
2450
+
2451
+ use super::{TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts};
2452
+
2453
+ use super::interface::{CharacterTokens, EOFToken, NullCharacterToken, ParseError};
2454
+ use super::interface::{EndTag, StartTag, Tag, TagKind};
2455
+ use super::interface::{TagToken, Token};
2456
+
2457
+ use markup5ever::buffer_queue::BufferQueue;
2458
+ use std::cell::RefCell;
2459
+
2460
+ use crate::LocalName;
2461
+
2462
+ // LinesMatch implements the TokenSink trait. It is used for testing to see
2463
+ // if current_line is being updated when process_token is called. The lines
2464
+ // vector is a collection of the line numbers that each token is on.
2465
+ struct LinesMatch {
2466
+ tokens: RefCell<Vec<Token>>,
2467
+ current_str: RefCell<StrTendril>,
2468
+ lines: RefCell<Vec<(Token, u64)>>,
2469
+ }
2470
+
2471
+ impl LinesMatch {
2472
+ fn new() -> LinesMatch {
2473
+ LinesMatch {
2474
+ tokens: RefCell::new(vec![]),
2475
+ current_str: RefCell::new(StrTendril::new()),
2476
+ lines: RefCell::new(vec![]),
2477
+ }
2478
+ }
2479
+
2480
+ fn push(&self, token: Token, line_number: u64) {
2481
+ self.finish_str();
2482
+ self.lines.borrow_mut().push((token, line_number));
2483
+ }
2484
+
2485
+ fn finish_str(&self) {
2486
+ if !self.current_str.borrow().is_empty() {
2487
+ let s = self.current_str.take();
2488
+ self.tokens.borrow_mut().push(CharacterTokens(s));
2489
+ }
2490
+ }
2491
+ }
2492
+
2493
+ impl TokenSink for LinesMatch {
2494
+ type Handle = ();
2495
+
2496
+ fn process_token(&self, token: Token, line_number: u64) -> TokenSinkResult<Self::Handle> {
2497
+ match token {
2498
+ CharacterTokens(b) => {
2499
+ self.current_str.borrow_mut().push_slice(&b);
2500
+ }
2501
+
2502
+ NullCharacterToken => {
2503
+ self.current_str.borrow_mut().push_char('\0');
2504
+ }
2505
+
2506
+ ParseError(_) => {
2507
+ panic!("unexpected parse error");
2508
+ }
2509
+
2510
+ TagToken(mut t) => {
2511
+ // The spec seems to indicate that one can emit
2512
+ // erroneous end tags with attrs, but the test
2513
+ // cases don't contain them.
2514
+ match t.kind {
2515
+ EndTag => {
2516
+ t.self_closing = false;
2517
+ t.attrs = vec![];
2518
+ }
2519
+ _ => t.attrs.sort_by(|a1, a2| a1.name.cmp(&a2.name)),
2520
+ }
2521
+ self.push(TagToken(t), line_number);
2522
+ }
2523
+
2524
+ EOFToken => (),
2525
+
2526
+ _ => self.push(token, line_number),
2527
+ }
2528
+ TokenSinkResult::Continue
2529
+ }
2530
+ }
2531
+
2532
+ // Take in tokens, process them, and return vector with line
2533
+ // numbers that each token is on
2534
+ fn tokenize(input: Vec<StrTendril>, opts: TokenizerOpts) -> Vec<(Token, u64)> {
2535
+ let sink = LinesMatch::new();
2536
+ let tok = Tokenizer::new(sink, opts);
2537
+ let buffer = BufferQueue::default();
2538
+ for chunk in input.into_iter() {
2539
+ buffer.push_back(chunk);
2540
+ let _ = tok.feed(&buffer);
2541
+ }
2542
+ tok.end();
2543
+ tok.sink.lines.take()
2544
+ }
2545
+
2546
+ // Create a tag token
2547
+ fn create_tag(token: StrTendril, tagkind: TagKind) -> Token {
2548
+ let name = LocalName::from(&*token);
2549
+
2550
+ TagToken(Tag {
2551
+ kind: tagkind,
2552
+ name,
2553
+ self_closing: false,
2554
+ attrs: vec![],
2555
+ })
2556
+ }
2557
+
2558
+ #[test]
2559
+ fn push_to_None_gives_singleton() {
2560
+ let mut s: Option<StrTendril> = None;
2561
+ option_push(&mut s, 'x');
2562
+ assert_eq!(s, Some("x".to_tendril()));
2563
+ }
2564
+
2565
+ #[test]
2566
+ fn push_to_empty_appends() {
2567
+ let mut s: Option<StrTendril> = Some(StrTendril::new());
2568
+ option_push(&mut s, 'x');
2569
+ assert_eq!(s, Some("x".to_tendril()));
2570
+ }
2571
+
2572
+ #[test]
2573
+ fn push_to_nonempty_appends() {
2574
+ let mut s: Option<StrTendril> = Some(StrTendril::from_slice("y"));
2575
+ option_push(&mut s, 'x');
2576
+ assert_eq!(s, Some("yx".to_tendril()));
2577
+ }
2578
+
2579
+ #[test]
2580
+ fn check_lines() {
2581
+ let opts = TokenizerOpts {
2582
+ exact_errors: false,
2583
+ discard_bom: true,
2584
+ profile: false,
2585
+ initial_state: None,
2586
+ last_start_tag_name: None,
2587
+ };
2588
+ let vector = vec![
2589
+ StrTendril::from("<a>\n"),
2590
+ StrTendril::from("<b>\n"),
2591
+ StrTendril::from("</b>\n"),
2592
+ StrTendril::from("</a>\n"),
2593
+ ];
2594
+ let expected = vec![
2595
+ (create_tag(StrTendril::from("a"), StartTag), 1),
2596
+ (create_tag(StrTendril::from("b"), StartTag), 2),
2597
+ (create_tag(StrTendril::from("b"), EndTag), 3),
2598
+ (create_tag(StrTendril::from("a"), EndTag), 4),
2599
+ ];
2600
+ let results = tokenize(vector, opts);
2601
+ assert_eq!(results, expected);
2602
+ }
2603
+
2604
+ #[test]
2605
+ fn check_lines_with_new_line() {
2606
+ let opts = TokenizerOpts {
2607
+ exact_errors: false,
2608
+ discard_bom: true,
2609
+ profile: false,
2610
+ initial_state: None,
2611
+ last_start_tag_name: None,
2612
+ };
2613
+ let vector = vec![
2614
+ StrTendril::from("<a>\r\n"),
2615
+ StrTendril::from("<b>\r\n"),
2616
+ StrTendril::from("</b>\r\n"),
2617
+ StrTendril::from("</a>\r\n"),
2618
+ ];
2619
+ let expected = vec![
2620
+ (create_tag(StrTendril::from("a"), StartTag), 1),
2621
+ (create_tag(StrTendril::from("b"), StartTag), 2),
2622
+ (create_tag(StrTendril::from("b"), EndTag), 3),
2623
+ (create_tag(StrTendril::from("a"), EndTag), 4),
2624
+ ];
2625
+ let results = tokenize(vector, opts);
2626
+ assert_eq!(results, expected);
2627
+ }
2628
+ }