@dvaji/cheerox 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/CONTRIBUTING.md +74 -0
  2. package/Cargo.lock +1094 -0
  3. package/Cargo.toml +46 -0
  4. package/LICENSE +21 -0
  5. package/README.md +121 -2
  6. package/RELEASING.md +114 -0
  7. package/THIRD_PARTY_NOTICES.md +7579 -0
  8. package/build.rs +3 -0
  9. package/cheerox.darwin-arm64.node +0 -0
  10. package/cheerox.darwin-x64.node +0 -0
  11. package/cheerox.linux-arm64-gnu.node +0 -0
  12. package/cheerox.linux-arm64-musl.node +0 -0
  13. package/cheerox.linux-x64-gnu.node +0 -0
  14. package/cheerox.linux-x64-musl.node +0 -0
  15. package/cheerox.win32-arm64-msvc.node +0 -0
  16. package/cheerox.win32-x64-msvc.node +0 -0
  17. package/dist/cheerio-compat-BS--qPzw.mjs +1648 -0
  18. package/dist/cheerio-compat-BS--qPzw.mjs.map +1 -0
  19. package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
  20. package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
  21. package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
  22. package/dist/cheerio-compat-DSwBAJ1s.cjs +1719 -0
  23. package/dist/cheerio-compat-DSwBAJ1s.cjs.map +1 -0
  24. package/dist/cheerio-compat.cjs +18 -0
  25. package/dist/cheerio-compat.d.cts +2 -0
  26. package/dist/cheerio-compat.d.mts +2 -0
  27. package/dist/cheerio-compat.d.ts +2 -0
  28. package/dist/cheerio-compat.mjs +1 -0
  29. package/dist/index.cjs +13 -0
  30. package/dist/index.cjs.map +1 -0
  31. package/dist/index.d.cts +28 -0
  32. package/dist/index.d.mts +6 -0
  33. package/dist/index.d.ts +28 -0
  34. package/dist/index.mjs +53 -0
  35. package/dist/index.mjs.map +1 -0
  36. package/dist/legacy-parse.cjs +5 -0
  37. package/dist/legacy-parse.d.mts +8 -0
  38. package/dist/legacy-parse.d.ts +7 -0
  39. package/dist/legacy-parse.mjs +1 -0
  40. package/dist/modern-B7hyW4cI.d.cts +102 -0
  41. package/dist/modern-B7hyW4cI.d.ts +102 -0
  42. package/dist/modern-CHQGP0Yt.cjs +2858 -0
  43. package/dist/modern-CHQGP0Yt.cjs.map +1 -0
  44. package/dist/modern-CJvSD_y5.d.mts +102 -0
  45. package/dist/modern-CvXcTn82.mjs +2769 -0
  46. package/dist/modern-CvXcTn82.mjs.map +1 -0
  47. package/dist/modern.cjs +19 -0
  48. package/dist/modern.d.cts +3 -0
  49. package/dist/modern.d.mts +3 -0
  50. package/dist/modern.d.ts +3 -0
  51. package/dist/modern.mjs +3 -0
  52. package/dist/nodes-jbWWhssj.cjs +4210 -0
  53. package/dist/nodes-jbWWhssj.cjs.map +1 -0
  54. package/dist/nodes-sYxb5qyZ.mjs +3803 -0
  55. package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
  56. package/dist/slim.cjs +16 -0
  57. package/dist/slim.cjs.map +1 -0
  58. package/dist/slim.d.cts +6 -0
  59. package/dist/slim.d.mts +6 -0
  60. package/dist/slim.d.ts +6 -0
  61. package/dist/slim.mjs +12 -0
  62. package/dist/slim.mjs.map +1 -0
  63. package/dist/types-CA01Zg3r.d.cts +526 -0
  64. package/dist/types-CA01Zg3r.d.mts +526 -0
  65. package/dist/types-CA01Zg3r.d.ts +526 -0
  66. package/dist/utils.cjs +31 -0
  67. package/dist/utils.cjs.map +1 -0
  68. package/dist/utils.d.cts +8 -0
  69. package/dist/utils.d.mts +8 -0
  70. package/dist/utils.d.ts +8 -0
  71. package/dist/utils.mjs +25 -0
  72. package/dist/utils.mjs.map +1 -0
  73. package/native/src/arena.rs +2109 -0
  74. package/native/src/attributes.rs +946 -0
  75. package/native/src/codec.rs +298 -0
  76. package/native/src/diagnostics.rs +1249 -0
  77. package/native/src/document_order.rs +711 -0
  78. package/native/src/extraction.rs +321 -0
  79. package/native/src/gc.rs +175 -0
  80. package/native/src/hook_sink.rs +801 -0
  81. package/native/src/hook_stream.rs +513 -0
  82. package/native/src/id_hash.rs +42 -0
  83. package/native/src/immutable.rs +333 -0
  84. package/native/src/lib.rs +4119 -0
  85. package/native/src/location.rs +392 -0
  86. package/native/src/markup_tokenizer.rs +668 -0
  87. package/native/src/runtime.rs +368 -0
  88. package/native/src/selector.rs +1596 -0
  89. package/native/src/sink.rs +274 -0
  90. package/native/src/snapshot.rs +672 -0
  91. package/native/src/stream.rs +334 -0
  92. package/native/src/tokenizer.rs +581 -0
  93. package/native/src/tracker.rs +232 -0
  94. package/native/vendor/README.md +26 -0
  95. package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
  96. package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
  97. package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
  98. package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
  99. package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
  100. package/native/vendor/html5ever-0.35.0/README.md +60 -0
  101. package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
  102. package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
  103. package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
  104. package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
  105. package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
  106. package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
  107. package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
  108. package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
  109. package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
  110. package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
  111. package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
  112. package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
  113. package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
  114. package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
  115. package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
  116. package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
  117. package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
  118. package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
  119. package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
  120. package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
  121. package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
  122. package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
  123. package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
  124. package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
  125. package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
  126. package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
  127. package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
  128. package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
  129. package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
  130. package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
  131. package/native/vendor/html5gum-0.8.4/README.md +110 -0
  132. package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
  133. package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
  134. package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
  135. package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
  136. package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
  137. package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
  138. package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
  139. package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
  140. package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
  141. package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
  142. package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
  143. package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
  144. package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
  145. package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
  146. package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
  147. package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
  148. package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
  149. package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
  150. package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
  151. package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
  152. package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
  153. package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
  154. package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
  155. package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
  156. package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
  157. package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
  158. package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
  159. package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
  160. package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
  161. package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
  162. package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
  163. package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
  164. package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
  165. package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
  166. package/package.json +167 -4
  167. package/scripts/check-native.mjs +15 -0
  168. package/scripts/check-release.mjs +234 -0
  169. package/scripts/fix-declarations.mjs +39 -0
  170. package/scripts/generate-notices.mjs +188 -0
  171. package/scripts/notices/README.md +13 -0
  172. package/scripts/notices/cheerio-MIT.txt +21 -0
  173. package/scripts/notices/html5ever-AUTHORS.txt +20 -0
  174. package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
  175. package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
  176. package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
  177. package/scripts/notices/htmlparser2-MIT.txt +18 -0
  178. package/scripts/notices/napi-rs-MIT.txt +43 -0
  179. package/scripts/notices/neon-MIT.txt +19 -0
  180. package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
  181. package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
  182. package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
  183. package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
  184. package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
  185. package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
  186. package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
  187. package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
  188. package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
  189. package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
  190. package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
  191. package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
  192. package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
  193. package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
  194. package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
  195. package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
  196. package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
  197. package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
  198. package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
  199. package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
  200. package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
  201. package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
  202. package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
  203. package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
  204. package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
  205. package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
  206. package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
  207. package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
  208. package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
  209. package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
  210. package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
  211. package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
  212. package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
  213. package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
  214. package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
  215. package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
  216. package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
  217. package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
  218. package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
  219. package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
  220. package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
  221. package/scripts/notices/runtime.json +1632 -0
  222. package/scripts/notices/sources.json +561 -0
  223. package/scripts/notices/xray-MIT.txt +21 -0
  224. package/src/attribute-object.ts +71 -0
  225. package/src/attributes.ts +308 -0
  226. package/src/cheerio-compat/css.ts +58 -0
  227. package/src/cheerio-compat/render.ts +106 -0
  228. package/src/cheerio-compat/selectors.ts +51 -0
  229. package/src/cheerio-compat/types.ts +52 -0
  230. package/src/cheerio-compat.ts +2055 -0
  231. package/src/codec.ts +96 -0
  232. package/src/dom.ts +87 -0
  233. package/src/extract-plan.ts +97 -0
  234. package/src/extract.ts +322 -0
  235. package/src/hooks.ts +369 -0
  236. package/src/index.ts +48 -0
  237. package/src/io.ts +491 -0
  238. package/src/match.ts +207 -0
  239. package/src/modern.ts +259 -0
  240. package/src/native.ts +149 -0
  241. package/src/nodes.ts +831 -0
  242. package/src/parse.ts +53 -0
  243. package/src/parser-types.ts +157 -0
  244. package/src/router.ts +1129 -0
  245. package/src/selection.ts +682 -0
  246. package/src/slim.ts +12 -0
  247. package/src/snapshot.ts +291 -0
  248. package/src/transport.ts +113 -0
  249. package/src/types.ts +186 -0
  250. package/src/utils.ts +29 -0
  251. package/src/workers.ts +21 -0
  252. package/tsconfig.json +15 -0
  253. package/tsdown.config.mjs +17 -0
@@ -0,0 +1,298 @@
1
+ //! Lossless JavaScript UTF-16 over valid Rust strings. Ordinary Unicode keeps
2
+ //! its original representation; lone surrogates use BMP private-use frames.
3
+ use napi::{Env, JsString, NapiRaw};
4
+ use serde_json::Value;
5
+
6
+ /// A string returned to JavaScript. V8 decodes UTF-8 slowly, so text is
7
+ /// handed over as Latin-1 when every character fits, which also keeps V8's
8
+ /// compact one-byte string, and as UTF-16 otherwise.
9
+ pub struct JsText(pub String);
10
+
11
+ impl napi::bindgen_prelude::ToNapiValue for JsText {
12
+ unsafe fn to_napi_value(
13
+ env: napi::sys::napi_env,
14
+ value: Self,
15
+ ) -> napi::Result<napi::sys::napi_value> {
16
+ Ok(js_string(&Env::from_raw(env), &value.0)?.raw())
17
+ }
18
+ }
19
+
20
+ pub(crate) fn js_string(env: &Env, text: &str) -> napi::Result<JsString> {
21
+ if text.is_ascii() {
22
+ env.create_string_latin1(text.as_bytes())
23
+ } else if let Some(latin1) = latin1_bytes(text) {
24
+ env.create_string_latin1(&latin1)
25
+ } else {
26
+ env.create_string_utf16(&utf16_units(text))
27
+ }
28
+ }
29
+
30
+ /// `text` in Latin-1, if every character is below U+0100: UTF-8 encodes
31
+ /// those with lead bytes below 0xC4.
32
+ pub(crate) fn latin1_bytes(text: &str) -> Option<Vec<u8>> {
33
+ if text.bytes().any(|byte| byte >= 0xC4) {
34
+ return None;
35
+ }
36
+ Some(text.chars().map(|character| character as u8).collect())
37
+ }
38
+
39
+ /// `text` as UTF-16, widening eight ASCII bytes at a time and decoding other
40
+ /// characters directly from their UTF-8 bytes, which `text` guarantees valid.
41
+ pub(crate) fn utf16_units(text: &str) -> Vec<u16> {
42
+ const HIGH_BITS: u64 = 0x8080_8080_8080_8080;
43
+ let bytes = text.as_bytes();
44
+ let mut units = Vec::with_capacity(bytes.len());
45
+ let mut index = 0;
46
+ while index < bytes.len() {
47
+ if let Some(chunk) = bytes.get(index..index + 8) {
48
+ let word = u64::from_le_bytes(chunk.try_into().unwrap());
49
+ if word & HIGH_BITS == 0 {
50
+ units.extend(chunk.iter().map(|byte| u16::from(*byte)));
51
+ index += 8;
52
+ continue;
53
+ }
54
+ }
55
+ let lead = u32::from(bytes[index]);
56
+ let continuation = |offset: usize| u32::from(bytes[index + offset] & 0x3F);
57
+ let (scalar, length) = match lead {
58
+ 0x00..=0x7F => (lead, 1),
59
+ 0x80..=0xDF => (((lead & 0x1F) << 6) | continuation(1), 2),
60
+ 0xE0..=0xEF => (
61
+ ((lead & 0x0F) << 12) | (continuation(1) << 6) | continuation(2),
62
+ 3,
63
+ ),
64
+ _ => (
65
+ ((lead & 0x07) << 18)
66
+ | (continuation(1) << 12)
67
+ | (continuation(2) << 6)
68
+ | continuation(3),
69
+ 4,
70
+ ),
71
+ };
72
+ if scalar < 0x1_0000 {
73
+ units.push(scalar as u16);
74
+ } else {
75
+ let offset = scalar - 0x1_0000;
76
+ units.push(0xD800 | (offset >> 10) as u16);
77
+ units.push(0xDC00 | (offset & 0x3FF) as u16);
78
+ }
79
+ index += length;
80
+ }
81
+ units
82
+ }
83
+
84
+ pub(crate) const ESCAPE: char = '\u{e800}';
85
+ pub(crate) fn is_reserved(ch: char) -> bool {
86
+ ('\u{e000}'..=ESCAPE).contains(&ch)
87
+ }
88
+ pub(crate) fn has_encoding(value: &str) -> bool {
89
+ // Every reserved character starts with the lead byte 0xEE in UTF-8, so a
90
+ // byte search skips decoding the characters of ordinary text.
91
+ memchr::memchr_iter(0xEE, value.as_bytes())
92
+ .any(|index| value[index..].chars().next().is_some_and(is_reserved))
93
+ }
94
+ pub(crate) fn from_units(units: &[u16]) -> String {
95
+ if units.iter().all(|unit| !(0xd800..=0xe800).contains(unit)) {
96
+ return String::from_utf16_lossy(units);
97
+ }
98
+ let mut output = String::with_capacity(units.len());
99
+ let mut index = 0;
100
+ while index < units.len() {
101
+ let unit = units[index];
102
+ if (0xd800..=0xdbff).contains(&unit)
103
+ && units
104
+ .get(index + 1)
105
+ .is_some_and(|unit| (0xdc00..=0xdfff).contains(unit))
106
+ {
107
+ let code =
108
+ 0x10000 + ((unit as u32 - 0xd800) << 10) + (units[index + 1] as u32 - 0xdc00);
109
+ output.push(char::from_u32(code).unwrap());
110
+ index += 2;
111
+ continue;
112
+ }
113
+ if (0xd800..=0xdfff).contains(&unit) {
114
+ output.push(char::from_u32(0xe000 + unit as u32 - 0xd800).unwrap());
115
+ } else {
116
+ let ch = char::from_u32(unit as u32).unwrap();
117
+ if is_reserved(ch) {
118
+ output.push(ESCAPE);
119
+ }
120
+ output.push(ch);
121
+ }
122
+ index += 1;
123
+ }
124
+ output
125
+ }
126
+ /// Receives the UTF-16 units of a string from [`visit_units`].
127
+ pub(crate) trait UnitSink {
128
+ /// A run of ASCII bytes starting at byte offset `start`, one unit each.
129
+ fn ascii(&mut self, start: usize, run: &[u8]);
130
+ /// One unit and the byte offset of the character that starts there, or
131
+ /// `None` for the low half of a surrogate pair.
132
+ fn unit(&mut self, offset: Option<usize>, unit: u16);
133
+ }
134
+
135
+ impl UnitSink for Vec<u16> {
136
+ fn ascii(&mut self, _: usize, run: &[u8]) {
137
+ self.extend(run.iter().map(|value| u16::from(*value)));
138
+ }
139
+ fn unit(&mut self, _: Option<usize>, unit: u16) {
140
+ self.push(unit);
141
+ }
142
+ }
143
+
144
+ /// Feeds the UTF-16 units of `value` to `sink`. With `framed`, lossless frames
145
+ /// decode to their original units.
146
+ pub(crate) fn visit_units(value: &str, framed: bool, sink: &mut impl UnitSink) {
147
+ let bytes = value.as_bytes();
148
+ let mut byte = 0;
149
+ while byte < bytes.len() {
150
+ let ascii = bytes[byte..]
151
+ .iter()
152
+ .position(|value| !value.is_ascii())
153
+ .unwrap_or(bytes.len() - byte);
154
+ sink.ascii(byte, &bytes[byte..byte + ascii]);
155
+ byte += ascii;
156
+ let Some(ch) = value[byte..].chars().next() else {
157
+ break;
158
+ };
159
+ let start = byte;
160
+ byte += ch.len_utf8();
161
+ if framed && ch == ESCAPE {
162
+ if let Some(next) = value[byte..]
163
+ .chars()
164
+ .next()
165
+ .filter(|next| is_reserved(*next))
166
+ {
167
+ sink.unit(Some(start), next as u16);
168
+ byte += next.len_utf8();
169
+ continue;
170
+ }
171
+ }
172
+ if framed && ('\u{e000}'..ESCAPE).contains(&ch) {
173
+ sink.unit(Some(start), 0xd800 + (ch as u16 - 0xe000));
174
+ continue;
175
+ }
176
+ let mut encoded = [0; 2];
177
+ let encoded = ch.encode_utf16(&mut encoded);
178
+ sink.unit(Some(start), encoded[0]);
179
+ if let Some(low) = encoded.get(1) {
180
+ sink.unit(None, *low);
181
+ }
182
+ }
183
+ }
184
+ pub(crate) fn to_units(value: &str) -> Vec<u16> {
185
+ let mut output = Vec::with_capacity(value.len());
186
+ visit_units(value, has_encoding(value), &mut output);
187
+ output
188
+ }
189
+ pub(crate) fn to_usv(value: &str) -> String {
190
+ if has_encoding(value) {
191
+ String::from_utf16_lossy(&to_units(value))
192
+ } else {
193
+ value.to_owned()
194
+ }
195
+ }
196
+ pub(crate) fn encode_scalar(ch: char) -> String {
197
+ let mut output = String::new();
198
+ if is_reserved(ch) {
199
+ output.push(ESCAPE);
200
+ }
201
+ output.push(ch);
202
+ output
203
+ }
204
+ pub(crate) fn encode_literal(value: &str) -> String {
205
+ if !has_encoding(value) {
206
+ return value.to_owned();
207
+ }
208
+ let mut output = String::with_capacity(value.len());
209
+ for ch in value.chars() {
210
+ if is_reserved(ch) {
211
+ output.push(ESCAPE);
212
+ }
213
+ output.push(ch);
214
+ }
215
+ output
216
+ }
217
+
218
+ /// JSON.parse accepts lone escaped UTF-16 units. serde_json's String decoder
219
+ /// requires scalar values, so translate only JSON unicode escape tokens first.
220
+ /// Raw characters are already encoded by the native transport and stay intact.
221
+ pub(crate) fn parse_json(source: &str) -> serde_json::Result<Value> {
222
+ if !source.contains("\\u") {
223
+ return serde_json::from_str(source);
224
+ }
225
+ let mut output = String::with_capacity(source.len());
226
+ let bytes = source.as_bytes();
227
+ let mut index = 0;
228
+ let mut quoted = false;
229
+ while index < source.len() {
230
+ if bytes[index] == b'"' {
231
+ quoted = !quoted;
232
+ output.push('"');
233
+ index += 1;
234
+ continue;
235
+ }
236
+ if quoted && bytes[index] == b'\\' {
237
+ if bytes.get(index + 1) == Some(&b'u') && index + 6 <= source.len() {
238
+ if let Some(unit) = source
239
+ .get(index + 2..index + 6)
240
+ .and_then(|text| u16::from_str_radix(text, 16).ok())
241
+ {
242
+ if (0xd800..=0xdbff).contains(&unit)
243
+ && source.get(index + 6..index + 8) == Some("\\u")
244
+ {
245
+ if let Some(next) = source
246
+ .get(index + 8..index + 12)
247
+ .and_then(|text| u16::from_str_radix(text, 16).ok())
248
+ .filter(|next| (0xdc00..=0xdfff).contains(next))
249
+ {
250
+ let _ = next;
251
+ output.push_str(&source[index..index + 12]);
252
+ index += 12;
253
+ continue;
254
+ }
255
+ }
256
+ if (0xd800..=0xdfff).contains(&unit) || (0xe000..=0xe800).contains(&unit) {
257
+ output.push_str(&from_units(&[unit]));
258
+ index += 6;
259
+ continue;
260
+ }
261
+ }
262
+ }
263
+ output.push('\\');
264
+ index += 1;
265
+ if let Some(ch) = source[index..].chars().next() {
266
+ output.push(ch);
267
+ index += ch.len_utf8();
268
+ }
269
+ continue;
270
+ }
271
+ let ch = source[index..].chars().next().unwrap();
272
+ output.push(ch);
273
+ index += ch.len_utf8();
274
+ }
275
+ serde_json::from_str(&output)
276
+ }
277
+ #[cfg(test)]
278
+ mod tests {
279
+ use super::*;
280
+ #[test]
281
+ fn framing_preserves_all_utf16_units_and_literal_private_use() {
282
+ let units = [0xd800, 0x41, 0xdfff, 0xe000, 0xe800, 0xd83d, 0xde00];
283
+ assert_eq!(to_units(&from_units(&units)), units);
284
+ assert_eq!(
285
+ to_usv(&from_units(&units)),
286
+ "\u{fffd}A\u{fffd}\u{e000}\u{e800}😀"
287
+ );
288
+ }
289
+ #[test]
290
+ fn json_data_decoding_preserves_escaped_units_and_literal_escape_sequences() {
291
+ let value =
292
+ parse_json(r#"{"a":"\ud800","b":"\ue000","c":"\\ud800","d":"\ud83d\ude00"}"#).unwrap();
293
+ assert_eq!(to_units(value["a"].as_str().unwrap()), vec![0xd800]);
294
+ assert_eq!(to_units(value["b"].as_str().unwrap()), vec![0xe000]);
295
+ assert_eq!(value["c"], r"\ud800");
296
+ assert_eq!(value["d"], "😀");
297
+ }
298
+ }