@dvaji/cheerox 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1094 -0
  3. package/Cargo.toml +46 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7579 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BS--qPzw.mjs +1648 -0
  18. package/dist/cheerio-compat-BS--qPzw.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DSwBAJ1s.cjs +1719 -0
  23. package/dist/cheerio-compat-DSwBAJ1s.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4119 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +26 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/package.json +167 -4
  167. package/scripts/check-native.mjs +15 -0
  168. package/scripts/check-release.mjs +234 -0
  169. package/scripts/fix-declarations.mjs +39 -0
  170. package/scripts/generate-notices.mjs +188 -0
  171. package/scripts/notices/README.md +13 -0
  172. package/scripts/notices/cheerio-MIT.txt +21 -0
  173. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  174. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  175. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  176. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  177. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  178. package/scripts/notices/napi-rs-MIT.txt +43 -0
  179. package/scripts/notices/neon-MIT.txt +19 -0
  180. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  181. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  182. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  183. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  184. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  185. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  186. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  187. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  188. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  189. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  190. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  191. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  192. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  193. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  194. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  195. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  196. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  197. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  198. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  199. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  200. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  201. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  202. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  203. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  204. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  205. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  206. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  207. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  208. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  209. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  210. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  211. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  212. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  213. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  214. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  215. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  216. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  217. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  218. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  219. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  220. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  221. package/scripts/notices/runtime.json +1632 -0
  222. package/scripts/notices/sources.json +561 -0
  223. package/scripts/notices/xray-MIT.txt +21 -0
  224. package/src/attribute-object.ts +71 -0
  225. package/src/attributes.ts +308 -0
  226. package/src/cheerio-compat/css.ts +58 -0
  227. package/src/cheerio-compat/render.ts +106 -0
  228. package/src/cheerio-compat/selectors.ts +51 -0
  229. package/src/cheerio-compat/types.ts +52 -0
  230. package/src/cheerio-compat.ts +2055 -0
  231. package/src/codec.ts +96 -0
  232. package/src/dom.ts +87 -0
  233. package/src/extract-plan.ts +97 -0
  234. package/src/extract.ts +322 -0
  235. package/src/hooks.ts +369 -0
  236. package/src/index.ts +48 -0
  237. package/src/io.ts +491 -0
  238. package/src/match.ts +207 -0
  239. package/src/modern.ts +259 -0
  240. package/src/native.ts +149 -0
  241. package/src/nodes.ts +831 -0
  242. package/src/parse.ts +53 -0
  243. package/src/parser-types.ts +157 -0
  244. package/src/router.ts +1129 -0
  245. package/src/selection.ts +682 -0
  246. package/src/slim.ts +12 -0
  247. package/src/snapshot.ts +291 -0
  248. package/src/transport.ts +113 -0
  249. package/src/types.ts +186 -0
  250. package/src/utils.ts +29 -0
  251. package/src/workers.ts +21 -0
  252. package/tsconfig.json +15 -0
  253. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,668 @@
1
+ //! A port of htmlparser2's Tokenizer state machine over UTF-16 code units.
2
+ //!
3
+ //! Offsets are UTF-16 indices, as in htmlparser2. Text and attribute values are
4
+ //! reported as raw spans; the tree builder decodes them, which yields the same
5
+ //! text as htmlparser2's per-entity callbacks because adjacent text merges.
6
+
7
+ const TAB: u16 = 0x09;
8
+ const NEW_LINE: u16 = 0x0a;
9
+ const FORM_FEED: u16 = 0x0c;
10
+ const CARRIAGE_RETURN: u16 = 0x0d;
11
+ const SPACE: u16 = 0x20;
12
+ const EXCLAMATION_MARK: u16 = 0x21;
13
+ const DOUBLE_QUOTE: u16 = 0x22;
14
+ const AMP: u16 = 0x26;
15
+ const SINGLE_QUOTE: u16 = 0x27;
16
+ const DASH: u16 = 0x2d;
17
+ const SLASH: u16 = 0x2f;
18
+ const LT: u16 = 0x3c;
19
+ const EQ: u16 = 0x3d;
20
+ const GT: u16 = 0x3e;
21
+ const QUESTION_MARK: u16 = 0x3f;
22
+ const OPENING_SQUARE_BRACKET: u16 = 0x5b;
23
+
24
+ const CDATA: &[u8] = b"CDATA[";
25
+ const CDATA_END: &[u8] = b"]]>";
26
+ const COMMENT_END: &[u8] = b"-->";
27
+ const SCRIPT_END: &[u8] = b"</script";
28
+ const STYLE_END: &[u8] = b"</style";
29
+ const TITLE_END: &[u8] = b"</title";
30
+ const TEXTAREA_END: &[u8] = b"</textarea";
31
+ const XMP_END: &[u8] = b"</xmp";
32
+
33
+ #[derive(Debug, PartialEq, Eq)]
34
+ pub(crate) enum Token {
35
+ /// Raw text between `start` and `end`; `decode` marks entity decoding.
36
+ Text {
37
+ start: usize,
38
+ end: usize,
39
+ decode: bool,
40
+ },
41
+ OpenTagName {
42
+ start: usize,
43
+ end: usize,
44
+ },
45
+ OpenTagEnd {
46
+ end: usize,
47
+ },
48
+ SelfClosingTag {
49
+ end: usize,
50
+ },
51
+ AttributeName {
52
+ start: usize,
53
+ end: usize,
54
+ },
55
+ /// Raw attribute value data; `decode` marks entity decoding.
56
+ AttributeData {
57
+ start: usize,
58
+ end: usize,
59
+ decode: bool,
60
+ },
61
+ AttributeEnd {
62
+ end: usize,
63
+ },
64
+ CloseTag {
65
+ start: usize,
66
+ end: usize,
67
+ },
68
+ Comment {
69
+ start: usize,
70
+ end: usize,
71
+ end_offset: usize,
72
+ },
73
+ Cdata {
74
+ start: usize,
75
+ end: usize,
76
+ end_offset: usize,
77
+ },
78
+ Declaration {
79
+ start: usize,
80
+ end: usize,
81
+ },
82
+ ProcessingInstruction {
83
+ start: usize,
84
+ end: usize,
85
+ },
86
+ End,
87
+ }
88
+
89
+ #[derive(Clone, Copy, PartialEq, Eq)]
90
+ enum State {
91
+ Text,
92
+ BeforeTagName,
93
+ InTagName,
94
+ InSelfClosingTag,
95
+ BeforeClosingTagName,
96
+ InClosingTagName,
97
+ AfterClosingTagName,
98
+ BeforeAttributeName,
99
+ InAttributeName,
100
+ AfterAttributeName,
101
+ BeforeAttributeValue,
102
+ InAttributeValueDq,
103
+ InAttributeValueSq,
104
+ InAttributeValueNq,
105
+ BeforeDeclaration,
106
+ InDeclaration,
107
+ InProcessingInstruction,
108
+ BeforeComment,
109
+ CdataSequence,
110
+ InSpecialComment,
111
+ InCommentLike,
112
+ BeforeSpecialS,
113
+ BeforeSpecialT,
114
+ SpecialStartSequence,
115
+ InSpecialTag,
116
+ }
117
+
118
+ fn is_whitespace(c: u16) -> bool {
119
+ matches!(c, SPACE | NEW_LINE | TAB | FORM_FEED | CARRIAGE_RETURN)
120
+ }
121
+
122
+ fn is_end_of_tag_section(c: u16) -> bool {
123
+ c == SLASH || c == GT || is_whitespace(c)
124
+ }
125
+
126
+ fn is_ascii_alpha(c: u16) -> bool {
127
+ (u16::from(b'a')..=u16::from(b'z')).contains(&c)
128
+ || (u16::from(b'A')..=u16::from(b'Z')).contains(&c)
129
+ }
130
+
131
+ fn lower(c: u16) -> u16 {
132
+ c | 0x20
133
+ }
134
+
135
+ /// Tokenizes `units` and passes each token to `emit`, in htmlparser2's order.
136
+ pub(crate) fn tokenize(units: &[u16], xml: bool, decode: bool, emit: impl FnMut(Token)) {
137
+ let mut tokenizer = Tokenizer {
138
+ units,
139
+ xml,
140
+ decode,
141
+ emit,
142
+ state: State::Text,
143
+ section_start: Some(0),
144
+ index: 0,
145
+ is_special: false,
146
+ sequence: COMMENT_END,
147
+ sequence_index: 0,
148
+ };
149
+ tokenizer.run();
150
+ }
151
+
152
+ struct Tokenizer<'a, F: FnMut(Token)> {
153
+ units: &'a [u16],
154
+ xml: bool,
155
+ decode: bool,
156
+ emit: F,
157
+ state: State,
158
+ /// Start of the section being read; `None` matches htmlparser2's `-1`.
159
+ section_start: Option<usize>,
160
+ index: usize,
161
+ is_special: bool,
162
+ sequence: &'static [u8],
163
+ sequence_index: usize,
164
+ }
165
+
166
+ impl<F: FnMut(Token)> Tokenizer<'_, F> {
167
+ fn run(&mut self) {
168
+ while self.index < self.units.len() {
169
+ let c = self.units[self.index];
170
+ match self.state {
171
+ State::Text => self.state_text(c),
172
+ State::SpecialStartSequence => self.state_special_start_sequence(c),
173
+ State::InSpecialTag => self.state_in_special_tag(c),
174
+ State::CdataSequence => self.state_cdata_sequence(c),
175
+ State::InAttributeValueDq => self.state_in_attribute_value_quoted(c, DOUBLE_QUOTE),
176
+ State::InAttributeName => self.state_in_attribute_name(c),
177
+ State::InCommentLike => self.state_in_comment_like(c),
178
+ State::InSpecialComment => self.state_in_special_comment(c),
179
+ State::BeforeAttributeName => self.state_before_attribute_name(c),
180
+ State::InTagName => self.state_in_tag_name(c),
181
+ State::InClosingTagName => self.state_in_closing_tag_name(c),
182
+ State::BeforeTagName => self.state_before_tag_name(c),
183
+ State::AfterAttributeName => self.state_after_attribute_name(c),
184
+ State::InAttributeValueSq => self.state_in_attribute_value_quoted(c, SINGLE_QUOTE),
185
+ State::BeforeAttributeValue => self.state_before_attribute_value(c),
186
+ State::BeforeClosingTagName => self.state_before_closing_tag_name(c),
187
+ State::AfterClosingTagName => self.state_after_closing_tag_name(c),
188
+ State::BeforeSpecialS => self.state_before_special_s(c),
189
+ State::BeforeSpecialT => self.state_before_special_t(c),
190
+ State::InAttributeValueNq => self.state_in_attribute_value_unquoted(c),
191
+ State::InSelfClosingTag => self.state_in_self_closing_tag(c),
192
+ State::InDeclaration => self.state_in_declaration(c),
193
+ State::BeforeDeclaration => self.state_before_declaration(c),
194
+ State::BeforeComment => self.state_before_comment(c),
195
+ State::InProcessingInstruction => self.state_in_processing_instruction(c),
196
+ }
197
+ self.index += 1;
198
+ }
199
+ self.finish();
200
+ }
201
+
202
+ fn start(&self) -> usize {
203
+ self.section_start.unwrap_or(0)
204
+ }
205
+
206
+ fn emit_text(&mut self, start: usize, end: usize, decode: bool) {
207
+ if start < end {
208
+ (self.emit)(Token::Text { start, end, decode });
209
+ }
210
+ }
211
+
212
+ /// Advances to the next `c`; leaves `index` on the last unit when absent.
213
+ fn fast_forward_to(&mut self, c: u16) -> bool {
214
+ while self.index + 1 < self.units.len() {
215
+ self.index += 1;
216
+ if self.units[self.index] == c {
217
+ return true;
218
+ }
219
+ }
220
+ self.index = self.units.len().saturating_sub(1);
221
+ false
222
+ }
223
+
224
+ // Entities are decoded per span, so `&` never changes state and text can
225
+ // skip ahead to the next `<` whether or not entities are decoded.
226
+ fn state_text(&mut self, c: u16) {
227
+ if c == LT || self.fast_forward_to(LT) {
228
+ let start = self.start();
229
+ if self.index > start {
230
+ self.emit_text(start, self.index, self.decode);
231
+ }
232
+ self.state = State::BeforeTagName;
233
+ self.section_start = Some(self.index);
234
+ }
235
+ }
236
+
237
+ fn state_special_start_sequence(&mut self, c: u16) {
238
+ let is_end = self.sequence_index == self.sequence.len();
239
+ let is_match = if is_end {
240
+ is_end_of_tag_section(c)
241
+ } else {
242
+ lower(c) == u16::from(self.sequence[self.sequence_index])
243
+ };
244
+ if !is_match {
245
+ self.is_special = false;
246
+ } else if !is_end {
247
+ self.sequence_index += 1;
248
+ return;
249
+ }
250
+ self.sequence_index = 0;
251
+ self.state = State::InTagName;
252
+ self.state_in_tag_name(c);
253
+ }
254
+
255
+ fn state_in_special_tag(&mut self, c: u16) {
256
+ if self.sequence_index == self.sequence.len() {
257
+ if c == GT || is_whitespace(c) {
258
+ let end_of_text = self.index - self.sequence.len();
259
+ let start = self.start();
260
+ if start < end_of_text {
261
+ let is_title = self.sequence == TITLE_END;
262
+ self.emit_text(start, end_of_text, is_title && self.decode);
263
+ }
264
+ self.is_special = false;
265
+ self.section_start = Some(end_of_text + 2);
266
+ self.state_in_closing_tag_name(c);
267
+ return;
268
+ }
269
+ self.sequence_index = 0;
270
+ }
271
+ let is_title = self.sequence == TITLE_END;
272
+ if lower(c) == u16::from(self.sequence[self.sequence_index]) {
273
+ self.sequence_index += 1;
274
+ } else if self.sequence_index == 0 {
275
+ // htmlparser2 examines every unit of a title to decode entities.
276
+ if !is_title && self.fast_forward_to(LT) {
277
+ self.sequence_index = 1;
278
+ }
279
+ } else {
280
+ if is_title && self.decode && c == AMP {
281
+ // After a partial `</title` match, htmlparser2 consumes `&`
282
+ // without starting an entity, so it stays undecoded.
283
+ let start = self.start();
284
+ self.emit_text(start, self.index, true);
285
+ self.emit_text(self.index, self.index + 1, false);
286
+ self.section_start = Some(self.index + 1);
287
+ }
288
+ self.sequence_index = usize::from(c == LT);
289
+ }
290
+ }
291
+
292
+ fn state_cdata_sequence(&mut self, c: u16) {
293
+ if c == u16::from(CDATA[self.sequence_index]) {
294
+ self.sequence_index += 1;
295
+ if self.sequence_index == CDATA.len() {
296
+ self.state = State::InCommentLike;
297
+ self.sequence = CDATA_END;
298
+ self.sequence_index = 0;
299
+ self.section_start = Some(self.index + 1);
300
+ }
301
+ } else {
302
+ self.sequence_index = 0;
303
+ self.state = State::InDeclaration;
304
+ self.state_in_declaration(c);
305
+ }
306
+ }
307
+
308
+ fn state_in_comment_like(&mut self, c: u16) {
309
+ if c == u16::from(self.sequence[self.sequence_index]) {
310
+ self.sequence_index += 1;
311
+ if self.sequence_index == self.sequence.len() {
312
+ let start = self.start();
313
+ let token = if self.sequence == CDATA_END {
314
+ Token::Cdata {
315
+ start,
316
+ end: self.index,
317
+ end_offset: 2,
318
+ }
319
+ } else {
320
+ Token::Comment {
321
+ start,
322
+ end: self.index,
323
+ end_offset: 2,
324
+ }
325
+ };
326
+ (self.emit)(token);
327
+ self.sequence_index = 0;
328
+ self.section_start = Some(self.index + 1);
329
+ self.state = State::Text;
330
+ }
331
+ } else if self.sequence_index == 0 {
332
+ if self.fast_forward_to(u16::from(self.sequence[0])) {
333
+ self.sequence_index = 1;
334
+ }
335
+ } else if c != u16::from(self.sequence[self.sequence_index - 1]) {
336
+ self.sequence_index = 0;
337
+ }
338
+ }
339
+
340
+ /// HTML tag names start with an ASCII letter; XML allows anything that
341
+ /// does not end the tag section.
342
+ fn is_tag_start_char(&self, c: u16) -> bool {
343
+ if self.xml {
344
+ !is_end_of_tag_section(c)
345
+ } else {
346
+ is_ascii_alpha(c)
347
+ }
348
+ }
349
+
350
+ fn start_special(&mut self, sequence: &'static [u8], offset: usize) {
351
+ self.is_special = true;
352
+ self.sequence = sequence;
353
+ self.sequence_index = offset;
354
+ self.state = State::SpecialStartSequence;
355
+ }
356
+
357
+ fn state_before_tag_name(&mut self, c: u16) {
358
+ if c == EXCLAMATION_MARK {
359
+ self.state = State::BeforeDeclaration;
360
+ self.section_start = Some(self.index + 1);
361
+ } else if c == QUESTION_MARK {
362
+ self.state = State::InProcessingInstruction;
363
+ self.section_start = Some(self.index + 1);
364
+ } else if self.is_tag_start_char(c) {
365
+ let lowered = lower(c);
366
+ self.section_start = Some(self.index);
367
+ self.state = if self.xml {
368
+ State::InTagName
369
+ } else if lowered == u16::from(SCRIPT_END[2]) {
370
+ State::BeforeSpecialS
371
+ } else if lowered == u16::from(TITLE_END[2]) || lowered == u16::from(XMP_END[2]) {
372
+ State::BeforeSpecialT
373
+ } else {
374
+ State::InTagName
375
+ };
376
+ } else if c == SLASH {
377
+ self.state = State::BeforeClosingTagName;
378
+ } else {
379
+ self.state = State::Text;
380
+ self.state_text(c);
381
+ }
382
+ }
383
+
384
+ fn state_in_tag_name(&mut self, c: u16) {
385
+ if is_end_of_tag_section(c) {
386
+ let start = self.start();
387
+ (self.emit)(Token::OpenTagName {
388
+ start,
389
+ end: self.index,
390
+ });
391
+ self.section_start = None;
392
+ self.state = State::BeforeAttributeName;
393
+ self.state_before_attribute_name(c);
394
+ }
395
+ }
396
+
397
+ fn state_before_closing_tag_name(&mut self, c: u16) {
398
+ if c == GT {
399
+ self.state = State::Text;
400
+ } else if !is_whitespace(c) {
401
+ self.state = if self.is_tag_start_char(c) {
402
+ State::InClosingTagName
403
+ } else {
404
+ State::InSpecialComment
405
+ };
406
+ self.section_start = Some(self.index);
407
+ }
408
+ }
409
+
410
+ fn state_in_closing_tag_name(&mut self, c: u16) {
411
+ if c == GT || is_whitespace(c) {
412
+ let start = self.start();
413
+ (self.emit)(Token::CloseTag {
414
+ start,
415
+ end: self.index,
416
+ });
417
+ self.section_start = None;
418
+ self.state = State::AfterClosingTagName;
419
+ self.state_after_closing_tag_name(c);
420
+ }
421
+ }
422
+
423
+ fn state_after_closing_tag_name(&mut self, c: u16) {
424
+ if c == GT || self.fast_forward_to(GT) {
425
+ self.state = State::Text;
426
+ self.section_start = Some(self.index + 1);
427
+ }
428
+ }
429
+
430
+ fn state_before_attribute_name(&mut self, c: u16) {
431
+ if c == GT {
432
+ (self.emit)(Token::OpenTagEnd { end: self.index });
433
+ if self.is_special {
434
+ self.state = State::InSpecialTag;
435
+ self.sequence_index = 0;
436
+ } else {
437
+ self.state = State::Text;
438
+ }
439
+ self.section_start = Some(self.index + 1);
440
+ } else if c == SLASH {
441
+ self.state = State::InSelfClosingTag;
442
+ } else if !is_whitespace(c) {
443
+ self.state = State::InAttributeName;
444
+ self.section_start = Some(self.index);
445
+ }
446
+ }
447
+
448
+ fn state_in_self_closing_tag(&mut self, c: u16) {
449
+ if c == GT {
450
+ (self.emit)(Token::SelfClosingTag { end: self.index });
451
+ self.state = State::Text;
452
+ self.section_start = Some(self.index + 1);
453
+ self.is_special = false;
454
+ } else if !is_whitespace(c) {
455
+ self.state = State::BeforeAttributeName;
456
+ self.state_before_attribute_name(c);
457
+ }
458
+ }
459
+
460
+ fn state_in_attribute_name(&mut self, c: u16) {
461
+ if c == EQ || is_end_of_tag_section(c) {
462
+ let start = self.start();
463
+ (self.emit)(Token::AttributeName {
464
+ start,
465
+ end: self.index,
466
+ });
467
+ self.section_start = Some(self.index);
468
+ self.state = State::AfterAttributeName;
469
+ self.state_after_attribute_name(c);
470
+ }
471
+ }
472
+
473
+ fn state_after_attribute_name(&mut self, c: u16) {
474
+ if c == EQ {
475
+ self.state = State::BeforeAttributeValue;
476
+ } else if c == SLASH || c == GT {
477
+ let end = self.start();
478
+ (self.emit)(Token::AttributeEnd { end });
479
+ self.section_start = None;
480
+ self.state = State::BeforeAttributeName;
481
+ self.state_before_attribute_name(c);
482
+ } else if !is_whitespace(c) {
483
+ let end = self.start();
484
+ (self.emit)(Token::AttributeEnd { end });
485
+ self.state = State::InAttributeName;
486
+ self.section_start = Some(self.index);
487
+ }
488
+ }
489
+
490
+ fn state_before_attribute_value(&mut self, c: u16) {
491
+ if c == DOUBLE_QUOTE {
492
+ self.state = State::InAttributeValueDq;
493
+ self.section_start = Some(self.index + 1);
494
+ } else if c == SINGLE_QUOTE {
495
+ self.state = State::InAttributeValueSq;
496
+ self.section_start = Some(self.index + 1);
497
+ } else if !is_whitespace(c) {
498
+ self.section_start = Some(self.index);
499
+ self.state = State::InAttributeValueNq;
500
+ self.state_in_attribute_value_unquoted(c);
501
+ }
502
+ }
503
+
504
+ fn state_in_attribute_value_quoted(&mut self, c: u16, quote: u16) {
505
+ if c == quote || self.fast_forward_to(quote) {
506
+ let start = self.start();
507
+ (self.emit)(Token::AttributeData {
508
+ start,
509
+ end: self.index,
510
+ decode: self.decode,
511
+ });
512
+ self.section_start = None;
513
+ (self.emit)(Token::AttributeEnd {
514
+ end: self.index + 1,
515
+ });
516
+ self.state = State::BeforeAttributeName;
517
+ }
518
+ }
519
+
520
+ fn state_in_attribute_value_unquoted(&mut self, c: u16) {
521
+ if is_whitespace(c) || c == GT {
522
+ let start = self.start();
523
+ (self.emit)(Token::AttributeData {
524
+ start,
525
+ end: self.index,
526
+ decode: self.decode,
527
+ });
528
+ self.section_start = None;
529
+ (self.emit)(Token::AttributeEnd { end: self.index });
530
+ self.state = State::BeforeAttributeName;
531
+ self.state_before_attribute_name(c);
532
+ }
533
+ }
534
+
535
+ fn state_before_declaration(&mut self, c: u16) {
536
+ if c == OPENING_SQUARE_BRACKET {
537
+ self.state = State::CdataSequence;
538
+ self.sequence_index = 0;
539
+ } else {
540
+ self.state = if c == DASH {
541
+ State::BeforeComment
542
+ } else {
543
+ State::InDeclaration
544
+ };
545
+ }
546
+ }
547
+
548
+ fn state_in_declaration(&mut self, c: u16) {
549
+ if c == GT || self.fast_forward_to(GT) {
550
+ let start = self.start();
551
+ (self.emit)(Token::Declaration {
552
+ start,
553
+ end: self.index,
554
+ });
555
+ self.state = State::Text;
556
+ self.section_start = Some(self.index + 1);
557
+ }
558
+ }
559
+
560
+ fn state_in_processing_instruction(&mut self, c: u16) {
561
+ if c == GT || self.fast_forward_to(GT) {
562
+ let start = self.start();
563
+ (self.emit)(Token::ProcessingInstruction {
564
+ start,
565
+ end: self.index,
566
+ });
567
+ self.state = State::Text;
568
+ self.section_start = Some(self.index + 1);
569
+ }
570
+ }
571
+
572
+ fn state_before_comment(&mut self, c: u16) {
573
+ if c == DASH {
574
+ self.state = State::InCommentLike;
575
+ self.sequence = COMMENT_END;
576
+ // A comment may close immediately, as in `<!-->`.
577
+ self.sequence_index = 2;
578
+ self.section_start = Some(self.index + 1);
579
+ } else {
580
+ self.state = State::InDeclaration;
581
+ }
582
+ }
583
+
584
+ fn state_in_special_comment(&mut self, c: u16) {
585
+ if c == GT || self.fast_forward_to(GT) {
586
+ let start = self.start();
587
+ (self.emit)(Token::Comment {
588
+ start,
589
+ end: self.index,
590
+ end_offset: 0,
591
+ });
592
+ self.state = State::Text;
593
+ self.section_start = Some(self.index + 1);
594
+ }
595
+ }
596
+
597
+ fn state_before_special_s(&mut self, c: u16) {
598
+ let lowered = lower(c);
599
+ if lowered == u16::from(SCRIPT_END[3]) {
600
+ self.start_special(SCRIPT_END, 4);
601
+ } else if lowered == u16::from(STYLE_END[3]) {
602
+ self.start_special(STYLE_END, 4);
603
+ } else {
604
+ self.state = State::InTagName;
605
+ self.state_in_tag_name(c);
606
+ }
607
+ }
608
+
609
+ fn state_before_special_t(&mut self, c: u16) {
610
+ let lowered = lower(c);
611
+ if lowered == u16::from(TITLE_END[3]) {
612
+ self.start_special(TITLE_END, 4);
613
+ } else if lowered == u16::from(TEXTAREA_END[3]) {
614
+ self.start_special(TEXTAREA_END, 4);
615
+ } else if lowered == u16::from(XMP_END[3]) {
616
+ self.start_special(XMP_END, 4);
617
+ } else {
618
+ self.state = State::InTagName;
619
+ self.state_in_tag_name(c);
620
+ }
621
+ }
622
+
623
+ /// Mirrors htmlparser2's end-of-buffer cleanup followed by `finish`.
624
+ fn finish(&mut self) {
625
+ let end = self.units.len();
626
+ let start = self.start();
627
+ match self.state {
628
+ State::Text => self.emit_text(start, end, self.decode),
629
+ State::InSpecialTag => {
630
+ let decode = self.sequence == TITLE_END && self.decode;
631
+ self.emit_text(start, end, decode);
632
+ }
633
+ State::InAttributeValueDq | State::InAttributeValueSq | State::InAttributeValueNq => {
634
+ // An unfinished tag is dropped, as htmlparser2 does.
635
+ }
636
+ State::InCommentLike if start < end => {
637
+ let token = if self.sequence == CDATA_END {
638
+ Token::Cdata {
639
+ start,
640
+ end,
641
+ end_offset: 0,
642
+ }
643
+ } else {
644
+ Token::Comment {
645
+ start,
646
+ end,
647
+ end_offset: 0,
648
+ }
649
+ };
650
+ (self.emit)(token);
651
+ }
652
+ State::InTagName
653
+ | State::BeforeAttributeName
654
+ | State::BeforeAttributeValue
655
+ | State::AfterAttributeName
656
+ | State::InAttributeName
657
+ | State::InClosingTagName => {}
658
+ // Other states report their remaining input as undecoded text. An
659
+ // unset section start is -1 in htmlparser2, whose `slice(-1)` keeps
660
+ // only the final unit.
661
+ _ => {
662
+ let start = self.section_start.unwrap_or(end.saturating_sub(1));
663
+ self.emit_text(start, end, false);
664
+ }
665
+ }
666
+ (self.emit)(Token::End);
667
+ }
668
+ }