@dvaji/cheerox 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +74 -0
- package/Cargo.lock +1091 -0
- package/Cargo.toml +47 -0
- package/LICENSE +21 -0
- package/README.md +121 -2
- package/RELEASING.md +114 -0
- package/THIRD_PARTY_NOTICES.md +7581 -0
- package/build.rs +3 -0
- package/cheerox.darwin-arm64.node +0 -0
- package/cheerox.darwin-x64.node +0 -0
- package/cheerox.linux-arm64-gnu.node +0 -0
- package/cheerox.linux-arm64-musl.node +0 -0
- package/cheerox.linux-x64-gnu.node +0 -0
- package/cheerox.linux-x64-musl.node +0 -0
- package/cheerox.win32-arm64-msvc.node +0 -0
- package/cheerox.win32-x64-msvc.node +0 -0
- package/dist/cheerio-compat-BKkfJEAr.mjs +1648 -0
- package/dist/cheerio-compat-BKkfJEAr.mjs.map +1 -0
- package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
- package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
- package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
- package/dist/cheerio-compat-DcnYK9OB.cjs +1719 -0
- package/dist/cheerio-compat-DcnYK9OB.cjs.map +1 -0
- package/dist/cheerio-compat.cjs +18 -0
- package/dist/cheerio-compat.d.cts +2 -0
- package/dist/cheerio-compat.d.mts +2 -0
- package/dist/cheerio-compat.d.ts +2 -0
- package/dist/cheerio-compat.mjs +1 -0
- package/dist/index.cjs +13 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +28 -0
- package/dist/index.d.mts +6 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.mjs +53 -0
- package/dist/index.mjs.map +1 -0
- package/dist/legacy-parse.cjs +5 -0
- package/dist/legacy-parse.d.mts +8 -0
- package/dist/legacy-parse.d.ts +7 -0
- package/dist/legacy-parse.mjs +1 -0
- package/dist/modern-B7hyW4cI.d.cts +102 -0
- package/dist/modern-B7hyW4cI.d.ts +102 -0
- package/dist/modern-CHQGP0Yt.cjs +2858 -0
- package/dist/modern-CHQGP0Yt.cjs.map +1 -0
- package/dist/modern-CJvSD_y5.d.mts +102 -0
- package/dist/modern-CvXcTn82.mjs +2769 -0
- package/dist/modern-CvXcTn82.mjs.map +1 -0
- package/dist/modern.cjs +19 -0
- package/dist/modern.d.cts +3 -0
- package/dist/modern.d.mts +3 -0
- package/dist/modern.d.ts +3 -0
- package/dist/modern.mjs +3 -0
- package/dist/nodes-jbWWhssj.cjs +4210 -0
- package/dist/nodes-jbWWhssj.cjs.map +1 -0
- package/dist/nodes-sYxb5qyZ.mjs +3803 -0
- package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
- package/dist/slim.cjs +16 -0
- package/dist/slim.cjs.map +1 -0
- package/dist/slim.d.cts +6 -0
- package/dist/slim.d.mts +6 -0
- package/dist/slim.d.ts +6 -0
- package/dist/slim.mjs +12 -0
- package/dist/slim.mjs.map +1 -0
- package/dist/types-CA01Zg3r.d.cts +526 -0
- package/dist/types-CA01Zg3r.d.mts +526 -0
- package/dist/types-CA01Zg3r.d.ts +526 -0
- package/dist/utils.cjs +31 -0
- package/dist/utils.cjs.map +1 -0
- package/dist/utils.d.cts +8 -0
- package/dist/utils.d.mts +8 -0
- package/dist/utils.d.ts +8 -0
- package/dist/utils.mjs +25 -0
- package/dist/utils.mjs.map +1 -0
- package/native/src/arena.rs +2109 -0
- package/native/src/attributes.rs +946 -0
- package/native/src/codec.rs +298 -0
- package/native/src/diagnostics.rs +1249 -0
- package/native/src/document_order.rs +711 -0
- package/native/src/extraction.rs +321 -0
- package/native/src/gc.rs +175 -0
- package/native/src/hook_sink.rs +801 -0
- package/native/src/hook_stream.rs +513 -0
- package/native/src/id_hash.rs +42 -0
- package/native/src/immutable.rs +333 -0
- package/native/src/lib.rs +4120 -0
- package/native/src/location.rs +392 -0
- package/native/src/markup_tokenizer.rs +668 -0
- package/native/src/runtime.rs +368 -0
- package/native/src/selector.rs +1596 -0
- package/native/src/sink.rs +274 -0
- package/native/src/snapshot.rs +672 -0
- package/native/src/stream.rs +334 -0
- package/native/src/tokenizer.rs +581 -0
- package/native/src/tracker.rs +232 -0
- package/native/vendor/README.md +35 -0
- package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
- package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
- package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
- package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
- package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
- package/native/vendor/html5ever-0.35.0/README.md +60 -0
- package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
- package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
- package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
- package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
- package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
- package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
- package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
- package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
- package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
- package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
- package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
- package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
- package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
- package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
- package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
- package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
- package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
- package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
- package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
- package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
- package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
- package/native/vendor/html5gum-0.8.4/README.md +110 -0
- package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
- package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
- package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
- package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
- package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
- package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
- package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
- package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
- package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
- package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
- package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
- package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
- package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
- package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
- package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
- package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
- package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
- package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
- package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
- package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
- package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
- package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
- package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
- package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
- package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
- package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
- package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
- package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
- package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
- package/native/vendor/scraper-0.24.0/Cargo.toml +52 -0
- package/native/vendor/scraper-0.24.0/LICENSE +15 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/element.rs +212 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/mod.rs +236 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/serializable.rs +15 -0
- package/native/vendor/scraper-0.24.0/src/error/utils.rs +91 -0
- package/native/vendor/scraper-0.24.0/src/error.rs +118 -0
- package/native/vendor/scraper-0.24.0/src/html/mod.rs +270 -0
- package/native/vendor/scraper-0.24.0/src/html/serializable.rs +27 -0
- package/native/vendor/scraper-0.24.0/src/html/tree_sink.rs +285 -0
- package/native/vendor/scraper-0.24.0/src/lib.rs +180 -0
- package/native/vendor/scraper-0.24.0/src/node/serializable.rs +52 -0
- package/native/vendor/scraper-0.24.0/src/node.rs +465 -0
- package/native/vendor/scraper-0.24.0/src/selectable.rs +70 -0
- package/native/vendor/scraper-0.24.0/src/selector.rs +290 -0
- package/package.json +167 -4
- package/scripts/check-native.mjs +15 -0
- package/scripts/check-release.mjs +234 -0
- package/scripts/fix-declarations.mjs +39 -0
- package/scripts/generate-notices.mjs +194 -0
- package/scripts/notices/README.md +13 -0
- package/scripts/notices/cheerio-MIT.txt +21 -0
- package/scripts/notices/html5ever-AUTHORS.txt +20 -0
- package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
- package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
- package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
- package/scripts/notices/htmlparser2-MIT.txt +18 -0
- package/scripts/notices/napi-rs-MIT.txt +43 -0
- package/scripts/notices/neon-MIT.txt +19 -0
- package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
- package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
- package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
- package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
- package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
- package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
- package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
- package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
- package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
- package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
- package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
- package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
- package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
- package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
- package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
- package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
- package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
- package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
- package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
- package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
- package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
- package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
- package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
- package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
- package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
- package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
- package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
- package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
- package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
- package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
- package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
- package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
- package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
- package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
- package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
- package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
- package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
- package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
- package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
- package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
- package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
- package/scripts/notices/runtime.json +1632 -0
- package/scripts/notices/sources.json +561 -0
- package/scripts/notices/xray-MIT.txt +21 -0
- package/src/attribute-object.ts +71 -0
- package/src/attributes.ts +308 -0
- package/src/cheerio-compat/css.ts +58 -0
- package/src/cheerio-compat/render.ts +106 -0
- package/src/cheerio-compat/selectors.ts +51 -0
- package/src/cheerio-compat/types.ts +52 -0
- package/src/cheerio-compat.ts +2055 -0
- package/src/codec.ts +96 -0
- package/src/dom.ts +87 -0
- package/src/extract-plan.ts +97 -0
- package/src/extract.ts +322 -0
- package/src/hooks.ts +369 -0
- package/src/index.ts +48 -0
- package/src/io.ts +491 -0
- package/src/match.ts +207 -0
- package/src/modern.ts +259 -0
- package/src/native.ts +149 -0
- package/src/nodes.ts +831 -0
- package/src/parse.ts +53 -0
- package/src/parser-types.ts +157 -0
- package/src/router.ts +1129 -0
- package/src/selection.ts +682 -0
- package/src/slim.ts +12 -0
- package/src/snapshot.ts +291 -0
- package/src/transport.ts +113 -0
- package/src/types.ts +186 -0
- package/src/utils.ts +29 -0
- package/src/workers.ts +21 -0
- package/tsconfig.json +15 -0
- package/tsdown.config.mjs +17 -0
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
use std::cmp::min;
|
|
2
|
+
use std::convert::Infallible;
|
|
3
|
+
use std::fmt::Debug;
|
|
4
|
+
use std::fs::File;
|
|
5
|
+
use std::io::{self, Read};
|
|
6
|
+
|
|
7
|
+
/// An object that provides characters to the tokenizer.
|
|
8
|
+
///
|
|
9
|
+
/// See [`crate::Tokenizer::new`] for more information.
|
|
10
|
+
pub trait Reader {
|
|
11
|
+
/// The error returned by this reader.
|
|
12
|
+
type Error: std::error::Error;
|
|
13
|
+
|
|
14
|
+
/// Return a new byte from the input stream.
|
|
15
|
+
///
|
|
16
|
+
/// The input stream does **not** have to be preprocessed in any way, it can contain standalone
|
|
17
|
+
/// surrogates and have inconsistent newlines.
|
|
18
|
+
fn read_byte(&mut self) -> Result<Option<u8>, Self::Error>;
|
|
19
|
+
|
|
20
|
+
/// Attempt to read an entire string at once, either case-insensitively or not.
|
|
21
|
+
///
|
|
22
|
+
/// `case_sensitive=false` means that characters of the input stream should be compared while
|
|
23
|
+
/// ignoring ASCII-casing.
|
|
24
|
+
///
|
|
25
|
+
/// It can be assumed that this function is never called with a string that contains `\r` or
|
|
26
|
+
/// `\n`.
|
|
27
|
+
///
|
|
28
|
+
/// If the next characters equal to `s`, this function consumes the respective characters from
|
|
29
|
+
/// the input stream and returns `true`. If not, it does nothing and returns `false`.
|
|
30
|
+
fn try_read_string(&mut self, s: &[u8], case_sensitive: bool) -> Result<bool, Self::Error>;
|
|
31
|
+
|
|
32
|
+
/// Read an arbitrary amount of characters up until and including the next character that
|
|
33
|
+
/// matches an array entry in `needle`.
|
|
34
|
+
///
|
|
35
|
+
/// Return either:
|
|
36
|
+
///
|
|
37
|
+
/// 1. A chunk of consumed characters that does not contain any characters from `needle`. The chunk can be arbitrarily large or small.
|
|
38
|
+
/// 2. If the next character is included in `needle`, a string with just that character and nothing else.
|
|
39
|
+
///
|
|
40
|
+
/// In other words, case 1 means "we didn't find the needle yet, but here's some read data",
|
|
41
|
+
/// while case 2 means "we have found the needle".
|
|
42
|
+
///
|
|
43
|
+
/// The default implementation simply reads one character and calls `read_cb` with that
|
|
44
|
+
/// character, ignoring the needle entirely. It is recommended to manually implement
|
|
45
|
+
/// `read_until` if there is any sort of in-memory buffer where some sort of efficient string
|
|
46
|
+
/// search (see `memchr` or `jetscii` crate) can be run on.
|
|
47
|
+
///
|
|
48
|
+
/// The return value is usually borrowed from underlying buffers. If that's not possible, a
|
|
49
|
+
/// small buffer is provided as `char_buf` to put a single character into.
|
|
50
|
+
///
|
|
51
|
+
/// # Example
|
|
52
|
+
///
|
|
53
|
+
/// Here is how [`StringReader`] behaves:
|
|
54
|
+
///
|
|
55
|
+
/// ```rust
|
|
56
|
+
/// use html5gum::{Reader, Readable};
|
|
57
|
+
///
|
|
58
|
+
/// let mut reader = "hello world".to_reader();
|
|
59
|
+
/// let mut eof = false;
|
|
60
|
+
/// let mut chunks = Vec::new();
|
|
61
|
+
/// while !eof {
|
|
62
|
+
/// let mut char_buf = [0; 4];
|
|
63
|
+
/// let xs = reader.read_until(&[b' ', b'r'], &mut char_buf).unwrap();
|
|
64
|
+
/// if let Some(xs) = xs {
|
|
65
|
+
/// chunks.push(std::str::from_utf8(xs).unwrap().to_owned());
|
|
66
|
+
/// } else {
|
|
67
|
+
/// eof = true;
|
|
68
|
+
/// }
|
|
69
|
+
/// }
|
|
70
|
+
///
|
|
71
|
+
/// assert_eq!(chunks, &["hello", " ", "wo", "r", "ld"]);
|
|
72
|
+
/// ```
|
|
73
|
+
///
|
|
74
|
+
/// The inefficient default implementation produces:
|
|
75
|
+
///
|
|
76
|
+
/// ```text
|
|
77
|
+
/// ["h", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"]
|
|
78
|
+
/// ```
|
|
79
|
+
#[inline(always)]
|
|
80
|
+
fn read_until<'b>(
|
|
81
|
+
&'b mut self,
|
|
82
|
+
needle: &[u8],
|
|
83
|
+
char_buf: &'b mut [u8; 4],
|
|
84
|
+
) -> Result<Option<&'b [u8]>, Self::Error> {
|
|
85
|
+
let _ = needle;
|
|
86
|
+
|
|
87
|
+
match self.read_byte()? {
|
|
88
|
+
Some(x) => {
|
|
89
|
+
char_buf[0] = x;
|
|
90
|
+
Ok(Some(&char_buf[..1]))
|
|
91
|
+
}
|
|
92
|
+
None => Ok(None),
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/// An object that can be converted into a [`crate::Reader`].
|
|
98
|
+
///
|
|
99
|
+
/// For example, any utf8-string can be converted into a `StringReader`, such that
|
|
100
|
+
/// `Tokenizer::new("mystring")` and `Tokenizer::new(&String::new("foo"))` work.
|
|
101
|
+
pub trait Readable<'a> {
|
|
102
|
+
/// The reader type to which this type should be converted.
|
|
103
|
+
type Reader: Reader + 'a;
|
|
104
|
+
|
|
105
|
+
/// Convert self to some sort of reader.
|
|
106
|
+
fn to_reader(self) -> Self::Reader;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
impl<'a, R: 'a + Reader> Readable<'a> for R {
|
|
110
|
+
type Reader = Self;
|
|
111
|
+
|
|
112
|
+
fn to_reader(self) -> Self::Reader {
|
|
113
|
+
self
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/// A helper struct to seek forwards and backwards in strings. Used by the tokenizer to read HTML
|
|
118
|
+
/// from strings.
|
|
119
|
+
///
|
|
120
|
+
/// Example:
|
|
121
|
+
///
|
|
122
|
+
/// ```rust
|
|
123
|
+
/// use std::fmt::Write;
|
|
124
|
+
/// use html5gum::{Tokenizer, Token};
|
|
125
|
+
///
|
|
126
|
+
/// let html = "<title >hello world</title>";
|
|
127
|
+
/// let mut new_html = String::new();
|
|
128
|
+
///
|
|
129
|
+
/// for Ok(token) in Tokenizer::new(html) {
|
|
130
|
+
/// match token {
|
|
131
|
+
/// Token::StartTag(tag) => {
|
|
132
|
+
/// write!(new_html, "<{}>", String::from_utf8_lossy(&tag.name)).unwrap();
|
|
133
|
+
/// }
|
|
134
|
+
/// Token::String(hello_world) => {
|
|
135
|
+
/// write!(new_html, "{}", String::from_utf8_lossy(&hello_world)).unwrap();
|
|
136
|
+
/// }
|
|
137
|
+
/// Token::EndTag(tag) => {
|
|
138
|
+
/// write!(new_html, "</{}>", String::from_utf8_lossy(&tag.name)).unwrap();
|
|
139
|
+
/// }
|
|
140
|
+
/// _ => panic!("unexpected input"),
|
|
141
|
+
/// }
|
|
142
|
+
/// }
|
|
143
|
+
///
|
|
144
|
+
/// assert_eq!(new_html, "<title>hello world</title>");
|
|
145
|
+
/// ```
|
|
146
|
+
#[derive(Debug)]
|
|
147
|
+
pub struct StringReader<'a> {
|
|
148
|
+
input: &'a [u8],
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
impl<'a> StringReader<'a> {
|
|
152
|
+
fn new(input: &'a [u8]) -> Self {
|
|
153
|
+
StringReader { input }
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
impl<'a> Reader for StringReader<'a> {
|
|
158
|
+
type Error = Infallible;
|
|
159
|
+
|
|
160
|
+
#[inline(always)]
|
|
161
|
+
fn read_byte(&mut self) -> Result<Option<u8>, Self::Error> {
|
|
162
|
+
if self.input.is_empty() {
|
|
163
|
+
Ok(None)
|
|
164
|
+
} else {
|
|
165
|
+
let rv = self.input[0];
|
|
166
|
+
self.input = &self.input[1..];
|
|
167
|
+
Ok(Some(rv))
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
#[inline(always)]
|
|
172
|
+
fn read_until<'b>(
|
|
173
|
+
&'b mut self,
|
|
174
|
+
needle: &[u8],
|
|
175
|
+
_: &'b mut [u8; 4],
|
|
176
|
+
) -> Result<Option<&'b [u8]>, Self::Error> {
|
|
177
|
+
if self.input.is_empty() {
|
|
178
|
+
return Ok(None);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if let Some(needle_pos) = fast_find(needle, self.input) {
|
|
182
|
+
if needle_pos == 0 {
|
|
183
|
+
let (rv, new_input) = self.input.split_at(1);
|
|
184
|
+
self.input = new_input;
|
|
185
|
+
Ok(Some(rv))
|
|
186
|
+
} else {
|
|
187
|
+
let (rv, new_input) = self.input.split_at(needle_pos);
|
|
188
|
+
self.input = new_input;
|
|
189
|
+
Ok(Some(rv))
|
|
190
|
+
}
|
|
191
|
+
} else {
|
|
192
|
+
let rv = self.input;
|
|
193
|
+
self.input = b"";
|
|
194
|
+
Ok(Some(rv))
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
#[inline(always)]
|
|
199
|
+
fn try_read_string(&mut self, s1: &[u8], case_sensitive: bool) -> Result<bool, Self::Error> {
|
|
200
|
+
// we do not need to call validate_char here because `s` hopefully does not contain invalid
|
|
201
|
+
// characters
|
|
202
|
+
if let Some(s2) = self.input.get(..s1.len()) {
|
|
203
|
+
if s1 == s2 || (!case_sensitive && s1.eq_ignore_ascii_case(s2)) {
|
|
204
|
+
self.input = &self.input[s1.len()..];
|
|
205
|
+
return Ok(true);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
Ok(false)
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
impl<'a> Readable<'a> for &'a str {
|
|
214
|
+
type Reader = StringReader<'a>;
|
|
215
|
+
|
|
216
|
+
fn to_reader(self) -> Self::Reader {
|
|
217
|
+
StringReader::new(self.as_bytes())
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
impl<'a> Readable<'a> for &'a String {
|
|
222
|
+
type Reader = StringReader<'a>;
|
|
223
|
+
|
|
224
|
+
fn to_reader(self) -> Self::Reader {
|
|
225
|
+
StringReader::new(self.as_bytes())
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
impl<'a> Readable<'a> for &'a Vec<u8> {
|
|
230
|
+
type Reader = StringReader<'a>;
|
|
231
|
+
|
|
232
|
+
fn to_reader(self) -> Self::Reader {
|
|
233
|
+
StringReader::new(self.as_slice())
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
impl<'a> Readable<'a> for &'a [u8] {
|
|
238
|
+
type Reader = StringReader<'a>;
|
|
239
|
+
|
|
240
|
+
fn to_reader(self) -> Self::Reader {
|
|
241
|
+
StringReader::new(self)
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/// A [`IoReader`] can be used to construct a tokenizer from any type that implements
|
|
246
|
+
/// `std::io::Read`.
|
|
247
|
+
///
|
|
248
|
+
/// Because of trait impl conflicts, `IoReader` needs to be explicitly constructed. The exception
|
|
249
|
+
/// to that is `File`, which can be directly passed to `Tokenizer::new`.
|
|
250
|
+
///
|
|
251
|
+
/// When passing `Read`-types into html5gum, no I/O buffering is required. html5gum maintains its
|
|
252
|
+
/// own read-buffer (16kb, heap-allocated) such that it can be accessed directly. Put more simply,
|
|
253
|
+
/// it's wasteful to wrap your `File` in a `std::io::BufReader` before passing it to html5gum.
|
|
254
|
+
///
|
|
255
|
+
/// Example:
|
|
256
|
+
///
|
|
257
|
+
/// ```rust
|
|
258
|
+
/// use std::fmt::Write;
|
|
259
|
+
/// use html5gum::{Token, IoReader, Tokenizer};
|
|
260
|
+
///
|
|
261
|
+
/// let tokenizer = Tokenizer::new(IoReader::new("<title>hello world</title>".as_bytes()));
|
|
262
|
+
/// // more realistically: Tokenizer::new(File::open("index.html")?)
|
|
263
|
+
/// // long-form: Tokenizer::new(IoReader::new(File::open("index.html")?))
|
|
264
|
+
///
|
|
265
|
+
/// let mut new_html = String::new();
|
|
266
|
+
///
|
|
267
|
+
/// for token in tokenizer {
|
|
268
|
+
/// let token = token.unwrap();
|
|
269
|
+
///
|
|
270
|
+
/// match token {
|
|
271
|
+
/// Token::StartTag(tag) => {
|
|
272
|
+
/// write!(new_html, "<{}>", String::from_utf8_lossy(&tag.name)).unwrap();
|
|
273
|
+
/// }
|
|
274
|
+
/// Token::String(hello_world) => {
|
|
275
|
+
/// write!(new_html, "{}", String::from_utf8_lossy(&hello_world)).unwrap();
|
|
276
|
+
/// }
|
|
277
|
+
/// Token::EndTag(tag) => {
|
|
278
|
+
/// write!(new_html, "</{}>", String::from_utf8_lossy(&tag.name)).unwrap();
|
|
279
|
+
/// }
|
|
280
|
+
/// _ => panic!("unexpected input"),
|
|
281
|
+
/// }
|
|
282
|
+
///
|
|
283
|
+
/// }
|
|
284
|
+
///
|
|
285
|
+
/// assert_eq!(new_html, "<title>hello world</title>");
|
|
286
|
+
/// ```
|
|
287
|
+
#[derive(Debug)]
|
|
288
|
+
pub struct IoReader<R: Read, Buffer: AsRef<[u8]> + AsMut<[u8]> = Box<[u8]>> {
|
|
289
|
+
buf: Buffer,
|
|
290
|
+
read_cursor: usize,
|
|
291
|
+
write_cursor: usize,
|
|
292
|
+
reader: R,
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
impl<R: Read> IoReader<R> {
|
|
296
|
+
/// Construct a new `BufReadReader` from any type that implements `Read`.
|
|
297
|
+
pub fn new(reader: R) -> Self {
|
|
298
|
+
Self::new_with_buffer_size::<16384>(reader)
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/// Construct a new `BufReadReader` with a specific internal buffer size.
|
|
302
|
+
///
|
|
303
|
+
/// `new` defaults to a heap-allocated buffer of size 16kB.
|
|
304
|
+
pub fn new_with_buffer_size<const BUF_SIZE: usize>(reader: R) -> Self {
|
|
305
|
+
Self::new_with_buffer_impl(reader, Box::new([0; BUF_SIZE]))
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
impl<'a, R: Read> IoReader<R, &'a mut [u8]> {
|
|
310
|
+
/// Instantiate `IoReader` with a custom kind of buffer.
|
|
311
|
+
///
|
|
312
|
+
/// Buffers do not need to be zero-initialized.
|
|
313
|
+
pub fn new_with_buffer(reader: R, buf: &'a mut [u8]) -> Self {
|
|
314
|
+
Self::new_with_buffer_impl(reader, buf)
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
impl<R: Read, Buffer: AsRef<[u8]> + AsMut<[u8]>> IoReader<R, Buffer> {
|
|
319
|
+
// new_with_buffer_impl is not exposed because we cannot use any kind of AsMut. It has to be
|
|
320
|
+
// one where we can be sure that the size of the buffer does not change with repeated calls to
|
|
321
|
+
// `as_mut()`. There are complex solutions to this sort of thing, but for now it seems simpler
|
|
322
|
+
// to allow either Box<[u8; _]> or &mut [u8], and nothing else.
|
|
323
|
+
//
|
|
324
|
+
// See discussion at https://users.rust-lang.org/t/cowmut-or-borrowed-owned-mutable-temp-buffers/96595
|
|
325
|
+
fn new_with_buffer_impl(reader: R, buf: Buffer) -> Self {
|
|
326
|
+
IoReader {
|
|
327
|
+
buf,
|
|
328
|
+
read_cursor: 0,
|
|
329
|
+
write_cursor: 0,
|
|
330
|
+
reader,
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/// Ensure that the buffer contains at leaast `min_read_len` bytes to read.
|
|
335
|
+
///
|
|
336
|
+
/// Shift all to-be-read buffer contents between `self.read_cursor` and `self.write_cursor` to
|
|
337
|
+
/// the beginning of the buffer, and read extra bytes if necessary.
|
|
338
|
+
#[inline(always)]
|
|
339
|
+
fn prepare_buf(&mut self, min_read_len: usize) -> Result<(), io::Error> {
|
|
340
|
+
let mut readable_len = self.write_cursor - self.read_cursor;
|
|
341
|
+
debug_assert!(min_read_len <= self.buf.as_mut().len());
|
|
342
|
+
debug_assert!(readable_len <= self.buf.as_mut().len());
|
|
343
|
+
if readable_len < min_read_len {
|
|
344
|
+
let mut raw_buf = &mut self.buf.as_mut()[..];
|
|
345
|
+
raw_buf.copy_within(self.read_cursor..self.write_cursor, 0);
|
|
346
|
+
raw_buf = &mut raw_buf[readable_len..];
|
|
347
|
+
while readable_len < min_read_len {
|
|
348
|
+
let n = self.reader.read(raw_buf)?;
|
|
349
|
+
if n == 0 {
|
|
350
|
+
break;
|
|
351
|
+
}
|
|
352
|
+
readable_len += n;
|
|
353
|
+
raw_buf = &mut raw_buf[n..];
|
|
354
|
+
}
|
|
355
|
+
self.write_cursor = readable_len;
|
|
356
|
+
self.read_cursor = 0;
|
|
357
|
+
}
|
|
358
|
+
Ok(())
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
impl<R: Read, Buffer: AsRef<[u8]> + AsMut<[u8]>> Reader for IoReader<R, Buffer> {
|
|
363
|
+
type Error = io::Error;
|
|
364
|
+
|
|
365
|
+
#[inline(always)]
|
|
366
|
+
fn read_byte(&mut self) -> Result<Option<u8>, Self::Error> {
|
|
367
|
+
self.prepare_buf(1)?;
|
|
368
|
+
if self.read_cursor == self.write_cursor {
|
|
369
|
+
return Ok(None);
|
|
370
|
+
}
|
|
371
|
+
let rv = self.buf.as_mut().get(self.read_cursor).copied();
|
|
372
|
+
if rv.is_some() {
|
|
373
|
+
self.read_cursor += 1;
|
|
374
|
+
}
|
|
375
|
+
Ok(rv)
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
#[inline(always)]
|
|
379
|
+
fn try_read_string(&mut self, s1: &[u8], case_sensitive: bool) -> Result<bool, Self::Error> {
|
|
380
|
+
debug_assert!(!s1.contains(&b'\r'));
|
|
381
|
+
debug_assert!(!s1.contains(&b'\n'));
|
|
382
|
+
|
|
383
|
+
self.prepare_buf(s1.len())?;
|
|
384
|
+
let s2 = &self.buf.as_mut()
|
|
385
|
+
[self.read_cursor..min(self.read_cursor + s1.len(), self.write_cursor)];
|
|
386
|
+
if s1 == s2 || (!case_sensitive && s1.eq_ignore_ascii_case(s2)) {
|
|
387
|
+
self.read_cursor += s1.len();
|
|
388
|
+
Ok(true)
|
|
389
|
+
} else {
|
|
390
|
+
Ok(false)
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
#[inline(always)]
|
|
395
|
+
fn read_until<'b>(
|
|
396
|
+
&'b mut self,
|
|
397
|
+
needle: &[u8],
|
|
398
|
+
_: &'b mut [u8; 4],
|
|
399
|
+
) -> Result<Option<&'b [u8]>, Self::Error> {
|
|
400
|
+
self.prepare_buf(4)?;
|
|
401
|
+
let buf = &self.buf.as_ref()[self.read_cursor..self.write_cursor];
|
|
402
|
+
if buf.is_empty() {
|
|
403
|
+
Ok(None)
|
|
404
|
+
} else if let Some(needle_pos) = fast_find(needle, buf) {
|
|
405
|
+
if needle_pos == 0 {
|
|
406
|
+
self.read_cursor += 1;
|
|
407
|
+
Ok(Some(&buf[..1]))
|
|
408
|
+
} else {
|
|
409
|
+
self.read_cursor += needle_pos;
|
|
410
|
+
Ok(Some(&buf[..needle_pos]))
|
|
411
|
+
}
|
|
412
|
+
} else {
|
|
413
|
+
self.read_cursor += buf.len();
|
|
414
|
+
Ok(Some(buf))
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
impl<'a> Readable<'a> for File {
|
|
420
|
+
type Reader = IoReader<File>;
|
|
421
|
+
|
|
422
|
+
fn to_reader(self) -> Self::Reader {
|
|
423
|
+
IoReader::new(self)
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
#[inline]
|
|
428
|
+
fn fast_find(needle: &[u8], haystack: &[u8]) -> Option<usize> {
|
|
429
|
+
#[cfg(feature = "jetscii")]
|
|
430
|
+
{
|
|
431
|
+
debug_assert!(needle.len() <= 16);
|
|
432
|
+
let mut needle_arr = [0; 16];
|
|
433
|
+
needle_arr[..needle.len()].copy_from_slice(needle);
|
|
434
|
+
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
|
|
435
|
+
jetscii::Bytes::new(needle_arr, needle.len() as i32, |b| needle.contains(&b)).find(haystack)
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
#[cfg(not(feature = "jetscii"))]
|
|
439
|
+
haystack.iter().position(|b| needle.contains(b))
|
|
440
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/// A single bound of a [`Span`].
|
|
2
|
+
///
|
|
3
|
+
/// For example use `()` as a bound to ignore spans and use [`usize`] for the default
|
|
4
|
+
/// implementation.
|
|
5
|
+
pub trait SpanBound:
|
|
6
|
+
Sized
|
|
7
|
+
+ Clone
|
|
8
|
+
+ Copy
|
|
9
|
+
+ std::fmt::Debug
|
|
10
|
+
+ Default
|
|
11
|
+
+ PartialEq
|
|
12
|
+
+ Eq
|
|
13
|
+
+ PartialOrd
|
|
14
|
+
+ Ord
|
|
15
|
+
+ std::hash::Hash
|
|
16
|
+
{
|
|
17
|
+
/// Offset the bound by a given value.
|
|
18
|
+
#[must_use]
|
|
19
|
+
fn offset(self, by: isize) -> Self;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/// Position/ boundary `start..end` in the input.
|
|
23
|
+
///
|
|
24
|
+
/// The position will mostly be a byte offset, but depending on the [crate::Reader] it originates
|
|
25
|
+
/// from, it can be something entirely else.
|
|
26
|
+
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
|
|
27
|
+
pub struct Span<B: SpanBound = usize> {
|
|
28
|
+
/// Start position (inclusive) of the span.
|
|
29
|
+
pub start: B,
|
|
30
|
+
/// End position (exclusive) of the span.
|
|
31
|
+
pub end: B,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
impl Span<()> {
|
|
35
|
+
/// Dummy empty span for tests.
|
|
36
|
+
#[doc(hidden)]
|
|
37
|
+
pub const DUMMY: Self = Self { start: (), end: () };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
impl SpanBound for () {
|
|
41
|
+
fn offset(self, _by: isize) -> Self {}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
impl SpanBound for usize {
|
|
45
|
+
fn offset(self, by: isize) -> Self {
|
|
46
|
+
self.saturating_add_signed(by)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// A value together with its [`Span`].
|
|
51
|
+
///
|
|
52
|
+
/// This type implements [`Deref`](std::ops::Deref) and [`DerefMut`](std::ops::DerefMut),
|
|
53
|
+
/// allowing you to access the inner value directly without using `.value`:
|
|
54
|
+
///
|
|
55
|
+
/// ```
|
|
56
|
+
/// # use html5gum::Spanned;
|
|
57
|
+
/// let spanned: Spanned<String, ()> = "hello".to_string().into();
|
|
58
|
+
/// assert_eq!(spanned.len(), 5); // calls String::len() via Deref
|
|
59
|
+
/// assert_eq!(&*spanned, "hello"); // dereference to get &String
|
|
60
|
+
/// ```
|
|
61
|
+
#[allow(missing_docs)]
|
|
62
|
+
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
|
|
63
|
+
pub struct Spanned<T, B: SpanBound = usize> {
|
|
64
|
+
pub value: T,
|
|
65
|
+
pub span: Span<B>,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl<T, B: SpanBound> From<T> for Spanned<T, B> {
|
|
69
|
+
fn from(value: T) -> Self {
|
|
70
|
+
Self {
|
|
71
|
+
value,
|
|
72
|
+
span: Span::default(),
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
impl<B: SpanBound> From<Vec<u8>> for Spanned<crate::HtmlString, B> {
|
|
78
|
+
fn from(value: Vec<u8>) -> Self {
|
|
79
|
+
Self {
|
|
80
|
+
value: value.into(),
|
|
81
|
+
span: Span::default(),
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
impl<T, B: SpanBound> std::ops::Deref for Spanned<T, B> {
|
|
87
|
+
type Target = T;
|
|
88
|
+
|
|
89
|
+
fn deref(&self) -> &Self::Target {
|
|
90
|
+
&self.value
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
impl<T, B: SpanBound> std::ops::DerefMut for Spanned<T, B> {
|
|
95
|
+
fn deref_mut(&mut self) -> &mut Self::Target {
|
|
96
|
+
&mut self.value
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/// Tokenizer that the tokenizer can be switched to from within the emitter.
|
|
2
|
+
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
3
|
+
pub enum State {
|
|
4
|
+
/// The data state.
|
|
5
|
+
Data,
|
|
6
|
+
/// The plain text state.
|
|
7
|
+
PlainText,
|
|
8
|
+
/// The RC data state.
|
|
9
|
+
RcData,
|
|
10
|
+
/// The raw text state.
|
|
11
|
+
RawText,
|
|
12
|
+
/// The script data state.
|
|
13
|
+
ScriptData,
|
|
14
|
+
/// The cdata section state.
|
|
15
|
+
CdataSection,
|
|
16
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
//! Module of helper functions for integration tests.
|
|
2
|
+
//!
|
|
3
|
+
//! Those tests should only test public API surface in general, with some exceptions as provided by
|
|
4
|
+
//! this module.
|
|
5
|
+
use crate::Reader;
|
|
6
|
+
use std::cell::Cell;
|
|
7
|
+
use std::sync::atomic::{AtomicBool, Ordering};
|
|
8
|
+
|
|
9
|
+
/// Global flag to enable trace logging. When false, trace_log is a no-op.
|
|
10
|
+
///
|
|
11
|
+
/// During fuzzing, we want to run with cfg(debug_assertions) yet at the same time not run
|
|
12
|
+
/// trace_log to keep target stability (in AFL terms). Adding more cargo featureflags is unwieldy.
|
|
13
|
+
static TRACE_LOG_ENABLED: AtomicBool = AtomicBool::new(false);
|
|
14
|
+
|
|
15
|
+
thread_local! {
|
|
16
|
+
/// Buffer of all debugging output logged internally by html5gum.
|
|
17
|
+
pub static OUTPUT: Cell<String> = Cell::default();
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/// Enable trace logging for the current process.
|
|
21
|
+
/// This should be called at the start of any test that needs to capture trace logs.
|
|
22
|
+
pub fn enable_trace_log() {
|
|
23
|
+
TRACE_LOG_ENABLED.store(true, Ordering::Relaxed);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/// Disable trace logging for the current process.
|
|
27
|
+
pub fn disable_trace_log() {
|
|
28
|
+
TRACE_LOG_ENABLED.store(false, Ordering::Relaxed);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/// Simple debug logger for tests.
|
|
32
|
+
///
|
|
33
|
+
/// The test harness used by `tests/html5lib_tokenizer.rs` cannot capture stdout, see [libtest-mimic
|
|
34
|
+
/// issue #9](https://github.com/LukasKalbertodt/libtest-mimic/issues/9) -- this is much more performant
|
|
35
|
+
/// than println anyway though.
|
|
36
|
+
///
|
|
37
|
+
/// A noop version for non-test builds is implemented in src/lib.rs
|
|
38
|
+
pub fn trace_log(msg: &str) {
|
|
39
|
+
if !TRACE_LOG_ENABLED.load(Ordering::Relaxed) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
OUTPUT.with(|cell| {
|
|
44
|
+
let mut buf = cell.take();
|
|
45
|
+
buf.push_str(msg);
|
|
46
|
+
buf.push('\n');
|
|
47
|
+
|
|
48
|
+
if buf.len() > 20 * 1024 * 1024 {
|
|
49
|
+
buf.clear();
|
|
50
|
+
buf.push_str("[truncated output]\n");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
cell.set(buf);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// A kind of reader that implements `read_until` very poorly. Only available in tests
|
|
58
|
+
#[derive(Debug)]
|
|
59
|
+
pub struct SlowReader<R: Reader>(pub R);
|
|
60
|
+
|
|
61
|
+
impl<R: Reader> Reader for SlowReader<R> {
|
|
62
|
+
type Error = R::Error;
|
|
63
|
+
|
|
64
|
+
fn read_byte(&mut self) -> Result<Option<u8>, Self::Error> {
|
|
65
|
+
self.0.read_byte()
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
fn try_read_string(&mut self, s: &[u8], case_sensitive: bool) -> Result<bool, Self::Error> {
|
|
69
|
+
self.0.try_read_string(s, case_sensitive)
|
|
70
|
+
}
|
|
71
|
+
}
|