@dvaji/cheerox 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +74 -0
- package/Cargo.lock +1091 -0
- package/Cargo.toml +47 -0
- package/LICENSE +21 -0
- package/README.md +121 -2
- package/RELEASING.md +114 -0
- package/THIRD_PARTY_NOTICES.md +7581 -0
- package/build.rs +3 -0
- package/cheerox.darwin-arm64.node +0 -0
- package/cheerox.darwin-x64.node +0 -0
- package/cheerox.linux-arm64-gnu.node +0 -0
- package/cheerox.linux-arm64-musl.node +0 -0
- package/cheerox.linux-x64-gnu.node +0 -0
- package/cheerox.linux-x64-musl.node +0 -0
- package/cheerox.win32-arm64-msvc.node +0 -0
- package/cheerox.win32-x64-msvc.node +0 -0
- package/dist/cheerio-compat-BKkfJEAr.mjs +1648 -0
- package/dist/cheerio-compat-BKkfJEAr.mjs.map +1 -0
- package/dist/cheerio-compat-DRLPzBdX.d.mts +88 -0
- package/dist/cheerio-compat-DRsINTwV.d.cts +88 -0
- package/dist/cheerio-compat-DRsINTwV.d.ts +88 -0
- package/dist/cheerio-compat-DcnYK9OB.cjs +1719 -0
- package/dist/cheerio-compat-DcnYK9OB.cjs.map +1 -0
- package/dist/cheerio-compat.cjs +18 -0
- package/dist/cheerio-compat.d.cts +2 -0
- package/dist/cheerio-compat.d.mts +2 -0
- package/dist/cheerio-compat.d.ts +2 -0
- package/dist/cheerio-compat.mjs +1 -0
- package/dist/index.cjs +13 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +28 -0
- package/dist/index.d.mts +6 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.mjs +53 -0
- package/dist/index.mjs.map +1 -0
- package/dist/legacy-parse.cjs +5 -0
- package/dist/legacy-parse.d.mts +8 -0
- package/dist/legacy-parse.d.ts +7 -0
- package/dist/legacy-parse.mjs +1 -0
- package/dist/modern-B7hyW4cI.d.cts +102 -0
- package/dist/modern-B7hyW4cI.d.ts +102 -0
- package/dist/modern-CHQGP0Yt.cjs +2858 -0
- package/dist/modern-CHQGP0Yt.cjs.map +1 -0
- package/dist/modern-CJvSD_y5.d.mts +102 -0
- package/dist/modern-CvXcTn82.mjs +2769 -0
- package/dist/modern-CvXcTn82.mjs.map +1 -0
- package/dist/modern.cjs +19 -0
- package/dist/modern.d.cts +3 -0
- package/dist/modern.d.mts +3 -0
- package/dist/modern.d.ts +3 -0
- package/dist/modern.mjs +3 -0
- package/dist/nodes-jbWWhssj.cjs +4210 -0
- package/dist/nodes-jbWWhssj.cjs.map +1 -0
- package/dist/nodes-sYxb5qyZ.mjs +3803 -0
- package/dist/nodes-sYxb5qyZ.mjs.map +1 -0
- package/dist/slim.cjs +16 -0
- package/dist/slim.cjs.map +1 -0
- package/dist/slim.d.cts +6 -0
- package/dist/slim.d.mts +6 -0
- package/dist/slim.d.ts +6 -0
- package/dist/slim.mjs +12 -0
- package/dist/slim.mjs.map +1 -0
- package/dist/types-CA01Zg3r.d.cts +526 -0
- package/dist/types-CA01Zg3r.d.mts +526 -0
- package/dist/types-CA01Zg3r.d.ts +526 -0
- package/dist/utils.cjs +31 -0
- package/dist/utils.cjs.map +1 -0
- package/dist/utils.d.cts +8 -0
- package/dist/utils.d.mts +8 -0
- package/dist/utils.d.ts +8 -0
- package/dist/utils.mjs +25 -0
- package/dist/utils.mjs.map +1 -0
- package/native/src/arena.rs +2109 -0
- package/native/src/attributes.rs +946 -0
- package/native/src/codec.rs +298 -0
- package/native/src/diagnostics.rs +1249 -0
- package/native/src/document_order.rs +711 -0
- package/native/src/extraction.rs +321 -0
- package/native/src/gc.rs +175 -0
- package/native/src/hook_sink.rs +801 -0
- package/native/src/hook_stream.rs +513 -0
- package/native/src/id_hash.rs +42 -0
- package/native/src/immutable.rs +333 -0
- package/native/src/lib.rs +4120 -0
- package/native/src/location.rs +392 -0
- package/native/src/markup_tokenizer.rs +668 -0
- package/native/src/runtime.rs +368 -0
- package/native/src/selector.rs +1596 -0
- package/native/src/sink.rs +274 -0
- package/native/src/snapshot.rs +672 -0
- package/native/src/stream.rs +334 -0
- package/native/src/tokenizer.rs +581 -0
- package/native/src/tracker.rs +232 -0
- package/native/vendor/README.md +35 -0
- package/native/vendor/html5ever-0.35.0/.cargo_vcs_info.json +6 -0
- package/native/vendor/html5ever-0.35.0/Cargo.lock +828 -0
- package/native/vendor/html5ever-0.35.0/Cargo.toml +85 -0
- package/native/vendor/html5ever-0.35.0/LICENSE-APACHE +201 -0
- package/native/vendor/html5ever-0.35.0/LICENSE-MIT +25 -0
- package/native/vendor/html5ever-0.35.0/README.md +60 -0
- package/native/vendor/html5ever-0.35.0/benches/html5ever.rs +86 -0
- package/native/vendor/html5ever-0.35.0/data/bench/lipsum-zh.html +19 -0
- package/native/vendor/html5ever-0.35.0/data/bench/lipsum.html +40 -0
- package/native/vendor/html5ever-0.35.0/data/bench/medium-fragment.html +24 -0
- package/native/vendor/html5ever-0.35.0/data/bench/small-fragment.html +7 -0
- package/native/vendor/html5ever-0.35.0/data/bench/strong.html +1 -0
- package/native/vendor/html5ever-0.35.0/data/bench/tiny-fragment.html +1 -0
- package/native/vendor/html5ever-0.35.0/examples/arena.rs +356 -0
- package/native/vendor/html5ever-0.35.0/examples/capi/tokenize.c +74 -0
- package/native/vendor/html5ever-0.35.0/examples/noop-tokenize.rs +47 -0
- package/native/vendor/html5ever-0.35.0/examples/noop-tree-builder.rs +132 -0
- package/native/vendor/html5ever-0.35.0/examples/print-tree-actions.rs +185 -0
- package/native/vendor/html5ever-0.35.0/examples/tokenize.rs +111 -0
- package/native/vendor/html5ever-0.35.0/src/driver.rs +148 -0
- package/native/vendor/html5ever-0.35.0/src/lib.rs +30 -0
- package/native/vendor/html5ever-0.35.0/src/macros.rs +34 -0
- package/native/vendor/html5ever-0.35.0/src/serialize/mod.rs +255 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/char_ref/mod.rs +530 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/interface.rs +99 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/mod.rs +2628 -0
- package/native/vendor/html5ever-0.35.0/src/tokenizer/states.rs +97 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/data.rs +170 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/mod.rs +1848 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/rules.rs +1494 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/tag_sets.rs +114 -0
- package/native/vendor/html5ever-0.35.0/src/tree_builder/types.rs +91 -0
- package/native/vendor/html5ever-0.35.0/src/util/str.rs +52 -0
- package/native/vendor/html5gum-0.8.4/Cargo.lock +1006 -0
- package/native/vendor/html5gum-0.8.4/Cargo.toml +109 -0
- package/native/vendor/html5gum-0.8.4/LICENSE +19 -0
- package/native/vendor/html5gum-0.8.4/README.md +110 -0
- package/native/vendor/html5gum-0.8.4/benches/files/0050.html +2131 -0
- package/native/vendor/html5gum-0.8.4/benches/files/6D30.html +1004 -0
- package/native/vendor/html5gum-0.8.4/benches/files.rs +32 -0
- package/native/vendor/html5gum-0.8.4/benches/patterns.rs +31 -0
- package/native/vendor/html5gum-0.8.4/src/arrayvec.rs +29 -0
- package/native/vendor/html5gum-0.8.4/src/char_validator.rs +134 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/callback.rs +676 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/default.rs +207 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/emitter.rs +344 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/html5ever.rs +306 -0
- package/native/vendor/html5gum-0.8.4/src/emitters/mod.rs +25 -0
- package/native/vendor/html5gum-0.8.4/src/entities.rs +3342 -0
- package/native/vendor/html5gum-0.8.4/src/error.rs +96 -0
- package/native/vendor/html5gum-0.8.4/src/htmlstring.rs +130 -0
- package/native/vendor/html5gum-0.8.4/src/lib.rs +66 -0
- package/native/vendor/html5gum-0.8.4/src/machine.rs +2136 -0
- package/native/vendor/html5gum-0.8.4/src/machine_helper.rs +277 -0
- package/native/vendor/html5gum-0.8.4/src/read_helper.rs +244 -0
- package/native/vendor/html5gum-0.8.4/src/reader.rs +440 -0
- package/native/vendor/html5gum-0.8.4/src/span.rs +98 -0
- package/native/vendor/html5gum-0.8.4/src/state.rs +16 -0
- package/native/vendor/html5gum-0.8.4/src/testutils.rs +71 -0
- package/native/vendor/html5gum-0.8.4/src/tokenizer.rs +189 -0
- package/native/vendor/html5gum-0.8.4/src/utils.rs +88 -0
- package/native/vendor/markup5ever-0.35.0/.cargo_vcs_info.json +6 -0
- package/native/vendor/markup5ever-0.35.0/Cargo.toml +45 -0
- package/native/vendor/markup5ever-0.35.0/LICENSE-APACHE +201 -0
- package/native/vendor/markup5ever-0.35.0/LICENSE-MIT +25 -0
- package/native/vendor/markup5ever-0.35.0/interface/mod.rs +422 -0
- package/native/vendor/markup5ever-0.35.0/interface/tree_builder.rs +280 -0
- package/native/vendor/markup5ever-0.35.0/lib.rs +59 -0
- package/native/vendor/markup5ever-0.35.0/serialize.rs +77 -0
- package/native/vendor/markup5ever-0.35.0/util/buffer_queue.rs +333 -0
- package/native/vendor/markup5ever-0.35.0/util/smallcharset.rs +88 -0
- package/native/vendor/scraper-0.24.0/Cargo.toml +52 -0
- package/native/vendor/scraper-0.24.0/LICENSE +15 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/element.rs +212 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/mod.rs +236 -0
- package/native/vendor/scraper-0.24.0/src/element_ref/serializable.rs +15 -0
- package/native/vendor/scraper-0.24.0/src/error/utils.rs +91 -0
- package/native/vendor/scraper-0.24.0/src/error.rs +118 -0
- package/native/vendor/scraper-0.24.0/src/html/mod.rs +270 -0
- package/native/vendor/scraper-0.24.0/src/html/serializable.rs +27 -0
- package/native/vendor/scraper-0.24.0/src/html/tree_sink.rs +285 -0
- package/native/vendor/scraper-0.24.0/src/lib.rs +180 -0
- package/native/vendor/scraper-0.24.0/src/node/serializable.rs +52 -0
- package/native/vendor/scraper-0.24.0/src/node.rs +465 -0
- package/native/vendor/scraper-0.24.0/src/selectable.rs +70 -0
- package/native/vendor/scraper-0.24.0/src/selector.rs +290 -0
- package/package.json +167 -4
- package/scripts/check-native.mjs +15 -0
- package/scripts/check-release.mjs +234 -0
- package/scripts/fix-declarations.mjs +39 -0
- package/scripts/generate-notices.mjs +194 -0
- package/scripts/notices/README.md +13 -0
- package/scripts/notices/cheerio-MIT.txt +21 -0
- package/scripts/notices/html5ever-AUTHORS.txt +20 -0
- package/scripts/notices/html5ever-COPYRIGHT.txt +8 -0
- package/scripts/notices/html5ever-LICENSE-APACHE.txt +201 -0
- package/scripts/notices/html5ever-LICENSE-MIT.txt +25 -0
- package/scripts/notices/htmlparser2-MIT.txt +18 -0
- package/scripts/notices/napi-rs-MIT.txt +43 -0
- package/scripts/notices/neon-MIT.txt +19 -0
- package/scripts/notices/runtime-01c266bced4a434da005.txt +3 -0
- package/scripts/notices/runtime-0a54e647fe54104658b5.txt +14 -0
- package/scripts/notices/runtime-0b74dfa0bcee5c420c6b.txt +25 -0
- package/scripts/notices/runtime-0d542e0c8804e39aa7f3.txt +177 -0
- package/scripts/notices/runtime-0f96a83840e146e43c0e.txt +21 -0
- package/scripts/notices/runtime-123a331b5dbf04c30097.txt +25 -0
- package/scripts/notices/runtime-172020dbfd5b53a226df.txt +34 -0
- package/scripts/notices/runtime-209fbbe0ad52d9235e37.txt +26 -0
- package/scripts/notices/runtime-23f18e03dc49df91622f.txt +23 -0
- package/scripts/notices/runtime-268872b9816f90fd8e85.txt +220 -0
- package/scripts/notices/runtime-35242e7a83f69875e6ed.txt +176 -0
- package/scripts/notices/runtime-378f5840b258e2779c39.txt +25 -0
- package/scripts/notices/runtime-4108245a1f2df9d4e94d.txt +25 -0
- package/scripts/notices/runtime-4455bf75a91154108304.txt +21 -0
- package/scripts/notices/runtime-553fffcd9b1cb158bc3e.txt +13 -0
- package/scripts/notices/runtime-62c7a1e35f56406896d7.txt +176 -0
- package/scripts/notices/runtime-6485b8ed310d3f0340bf.txt +25 -0
- package/scripts/notices/runtime-69fef7b0f322a6555415.txt +201 -0
- package/scripts/notices/runtime-6df43f6f4b5d4587f3d8.txt +187 -0
- package/scripts/notices/runtime-799e9ca9d179295ef372.txt +24 -0
- package/scripts/notices/runtime-7b63ecd5f1902af1b637.txt +25 -0
- package/scripts/notices/runtime-7bc7d5de2a0b793d4e1a.txt +21 -0
- package/scripts/notices/runtime-7e12e5df4bae12cb2158.txt +24 -0
- package/scripts/notices/runtime-861399f8c21c042b1105.txt +12 -0
- package/scripts/notices/runtime-8ada45cd9f843acf64e4.txt +201 -0
- package/scripts/notices/runtime-90eb64f0279b0d9432ac.txt +12 -0
- package/scripts/notices/runtime-a60eea817514531668d7.txt +201 -0
- package/scripts/notices/runtime-a7c936ff1ed8fa340172.txt +25 -0
- package/scripts/notices/runtime-ab6eec6caf0fa5775e41.txt +275 -0
- package/scripts/notices/runtime-b1181a40b2a7b25cf66f.txt +19 -0
- package/scripts/notices/runtime-b71bd43a069ca0641a9e.txt +25 -0
- package/scripts/notices/runtime-c16f8dcf1a368b83be78.txt +201 -0
- package/scripts/notices/runtime-c2cfccb812fe482101a8.txt +21 -0
- package/scripts/notices/runtime-e99d88d232bf57d70f0f.txt +25 -0
- package/scripts/notices/runtime-f32fb3b417a194167cfa.txt +9 -0
- package/scripts/notices/runtime-f5062c9a188d81dfe66b.txt +39 -0
- package/scripts/notices/runtime-f9070a832833fa7488fd.txt +4 -0
- package/scripts/notices/runtime-fab3dd6bdab226f1c086.txt +373 -0
- package/scripts/notices/runtime-fba3318d0c600380177e.txt +62 -0
- package/scripts/notices/runtime-ff8f68cb076caf8cefe7.txt +25 -0
- package/scripts/notices/runtime-ff92bed461f50338dd70.txt +74 -0
- package/scripts/notices/runtime.json +1632 -0
- package/scripts/notices/sources.json +561 -0
- package/scripts/notices/xray-MIT.txt +21 -0
- package/src/attribute-object.ts +71 -0
- package/src/attributes.ts +308 -0
- package/src/cheerio-compat/css.ts +58 -0
- package/src/cheerio-compat/render.ts +106 -0
- package/src/cheerio-compat/selectors.ts +51 -0
- package/src/cheerio-compat/types.ts +52 -0
- package/src/cheerio-compat.ts +2055 -0
- package/src/codec.ts +96 -0
- package/src/dom.ts +87 -0
- package/src/extract-plan.ts +97 -0
- package/src/extract.ts +322 -0
- package/src/hooks.ts +369 -0
- package/src/index.ts +48 -0
- package/src/io.ts +491 -0
- package/src/match.ts +207 -0
- package/src/modern.ts +259 -0
- package/src/native.ts +149 -0
- package/src/nodes.ts +831 -0
- package/src/parse.ts +53 -0
- package/src/parser-types.ts +157 -0
- package/src/router.ts +1129 -0
- package/src/selection.ts +682 -0
- package/src/slim.ts +12 -0
- package/src/snapshot.ts +291 -0
- package/src/transport.ts +113 -0
- package/src/types.ts +186 -0
- package/src/utils.ts +29 -0
- package/src/workers.ts +21 -0
- package/tsconfig.json +15 -0
- package/tsdown.config.mjs +17 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
//! One process-wide CPU budget, shared by batches and streaming parsers.
|
|
2
|
+
use napi::{Error, Result, Status};
|
|
3
|
+
use napi_derive::napi;
|
|
4
|
+
use rayon::prelude::*;
|
|
5
|
+
use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
|
|
6
|
+
use std::sync::{mpsc, Arc, Condvar, Mutex, OnceLock};
|
|
7
|
+
|
|
8
|
+
struct Configuration {
|
|
9
|
+
threads: usize,
|
|
10
|
+
started: bool,
|
|
11
|
+
pool: Option<Arc<rayon::ThreadPool>>,
|
|
12
|
+
error: Option<String>,
|
|
13
|
+
}
|
|
14
|
+
struct Runtime {
|
|
15
|
+
configuration: Mutex<Configuration>,
|
|
16
|
+
used: Mutex<u32>,
|
|
17
|
+
wake: Condvar,
|
|
18
|
+
batches: AtomicU32,
|
|
19
|
+
peak: AtomicU32,
|
|
20
|
+
}
|
|
21
|
+
fn runtime() -> &'static Runtime {
|
|
22
|
+
static SHARED: OnceLock<Runtime> = OnceLock::new();
|
|
23
|
+
SHARED.get_or_init(|| {
|
|
24
|
+
let default = std::thread::available_parallelism()
|
|
25
|
+
.map_or(1, usize::from)
|
|
26
|
+
.min(256);
|
|
27
|
+
let requested = std::env::var("CHEEROX_THREADS").ok();
|
|
28
|
+
let parsed = requested
|
|
29
|
+
.as_ref()
|
|
30
|
+
.and_then(|value| value.parse::<usize>().ok())
|
|
31
|
+
.filter(|value| (1..=256).contains(value));
|
|
32
|
+
Runtime {
|
|
33
|
+
configuration: Mutex::new(Configuration {
|
|
34
|
+
threads: parsed.unwrap_or(default),
|
|
35
|
+
started: false,
|
|
36
|
+
pool: None,
|
|
37
|
+
error: requested
|
|
38
|
+
.filter(|_| parsed.is_none())
|
|
39
|
+
.map(|_| "CHEEROX_THREADS must be an integer from 1 to 256".into()),
|
|
40
|
+
}),
|
|
41
|
+
used: Mutex::new(0),
|
|
42
|
+
wake: Condvar::new(),
|
|
43
|
+
batches: AtomicU32::new(0),
|
|
44
|
+
peak: AtomicU32::new(0),
|
|
45
|
+
}
|
|
46
|
+
})
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
#[napi(object)]
|
|
50
|
+
pub struct WorkerPoolInfo {
|
|
51
|
+
pub threads: u32,
|
|
52
|
+
pub active_batches: u32,
|
|
53
|
+
pub active_workers: u32,
|
|
54
|
+
pub peak_active_workers: u32,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
#[napi]
|
|
58
|
+
pub fn worker_pool_info() -> WorkerPoolInfo {
|
|
59
|
+
let shared = runtime();
|
|
60
|
+
WorkerPoolInfo {
|
|
61
|
+
threads: shared.configuration.lock().unwrap().threads as u32,
|
|
62
|
+
active_batches: shared.batches.load(Ordering::Relaxed),
|
|
63
|
+
active_workers: *shared.used.lock().unwrap(),
|
|
64
|
+
peak_active_workers: shared.peak.load(Ordering::Relaxed),
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#[napi]
|
|
69
|
+
pub fn configure_worker_pool(threads: u32) -> Result<WorkerPoolInfo> {
|
|
70
|
+
if !(1..=256).contains(&threads) {
|
|
71
|
+
return Err(Error::new(
|
|
72
|
+
Status::InvalidArg,
|
|
73
|
+
"Worker budget must be an integer from 1 to 256",
|
|
74
|
+
));
|
|
75
|
+
}
|
|
76
|
+
{
|
|
77
|
+
let mut config = runtime().configuration.lock().unwrap();
|
|
78
|
+
if config.started && config.threads != threads as usize {
|
|
79
|
+
return Err(Error::new(
|
|
80
|
+
Status::InvalidArg,
|
|
81
|
+
"Configure the worker budget before starting native asynchronous work",
|
|
82
|
+
));
|
|
83
|
+
}
|
|
84
|
+
if config.threads != threads as usize {
|
|
85
|
+
// A pool that only internal work has created yet is rebuilt at the new size.
|
|
86
|
+
config.pool = None;
|
|
87
|
+
}
|
|
88
|
+
config.threads = threads as usize;
|
|
89
|
+
config.error = None;
|
|
90
|
+
}
|
|
91
|
+
Ok(worker_pool_info())
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
pub(crate) struct Permit;
|
|
95
|
+
pub(crate) fn permit() -> Result<Permit> {
|
|
96
|
+
let shared = runtime();
|
|
97
|
+
let limit = {
|
|
98
|
+
let mut config = shared.configuration.lock().unwrap();
|
|
99
|
+
if let Some(error) = &config.error {
|
|
100
|
+
return Err(Error::new(Status::InvalidArg, error.clone()));
|
|
101
|
+
}
|
|
102
|
+
config.started = true;
|
|
103
|
+
config.threads as u32
|
|
104
|
+
};
|
|
105
|
+
let mut used = shared.used.lock().unwrap();
|
|
106
|
+
while *used >= limit {
|
|
107
|
+
used = shared.wake.wait(used).unwrap();
|
|
108
|
+
}
|
|
109
|
+
*used += 1;
|
|
110
|
+
shared.peak.fetch_max(*used, Ordering::Relaxed);
|
|
111
|
+
Ok(Permit)
|
|
112
|
+
}
|
|
113
|
+
impl Drop for Permit {
|
|
114
|
+
fn drop(&mut self) {
|
|
115
|
+
let shared = runtime();
|
|
116
|
+
*shared.used.lock().unwrap() -= 1;
|
|
117
|
+
shared.wake.notify_one();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/// The process-wide worker pool, created on first use with the configured size.
|
|
122
|
+
fn shared_pool() -> Result<Arc<rayon::ThreadPool>> {
|
|
123
|
+
pool(true)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/// The pool for work JavaScript did not ask for, which must not fix the
|
|
127
|
+
/// configured budget: the caller may configure it later.
|
|
128
|
+
fn internal_pool() -> Result<Arc<rayon::ThreadPool>> {
|
|
129
|
+
pool(false)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
fn pool(is_requested: bool) -> Result<Arc<rayon::ThreadPool>> {
|
|
133
|
+
let mut config = runtime().configuration.lock().unwrap();
|
|
134
|
+
if let Some(error) = &config.error {
|
|
135
|
+
return Err(Error::new(Status::InvalidArg, error.clone()));
|
|
136
|
+
}
|
|
137
|
+
config.started |= is_requested;
|
|
138
|
+
if config.pool.is_none() {
|
|
139
|
+
config.pool = Some(Arc::new(
|
|
140
|
+
rayon::ThreadPoolBuilder::new()
|
|
141
|
+
.num_threads(config.threads)
|
|
142
|
+
.build()
|
|
143
|
+
.map_err(|error| Error::new(Status::GenericFailure, error.to_string()))?,
|
|
144
|
+
));
|
|
145
|
+
}
|
|
146
|
+
Ok(config.pool.as_ref().unwrap().clone())
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// Values the pool has yet to drop. Past the limit they drop here, so busy
|
|
150
|
+
/// workers cannot let freed memory pile up.
|
|
151
|
+
static PENDING_RELEASES: AtomicU32 = AtomicU32::new(0);
|
|
152
|
+
const MAXIMUM_PENDING_RELEASES: u32 = 2;
|
|
153
|
+
|
|
154
|
+
/// Drops `value` on the shared pool or the release thread if either exists
|
|
155
|
+
/// and is keeping up, or here. It never starts either: finalizers call it
|
|
156
|
+
/// during garbage collection.
|
|
157
|
+
pub(crate) fn release_later<T: Send + 'static>(value: T) {
|
|
158
|
+
if PENDING_RELEASES.load(Ordering::Relaxed) >= MAXIMUM_PENDING_RELEASES {
|
|
159
|
+
return drop(value);
|
|
160
|
+
}
|
|
161
|
+
let pool = runtime().configuration.lock().unwrap().pool.clone();
|
|
162
|
+
if let Some(pool) = pool {
|
|
163
|
+
PENDING_RELEASES.fetch_add(1, Ordering::Relaxed);
|
|
164
|
+
return pool.spawn(move || {
|
|
165
|
+
drop(value);
|
|
166
|
+
PENDING_RELEASES.fetch_sub(1, Ordering::Relaxed);
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
if let Some(release_thread) = RELEASE_THREAD.get() {
|
|
170
|
+
PENDING_RELEASES.fetch_add(1, Ordering::Relaxed);
|
|
171
|
+
if let Err(unsent) = release_thread.send(Box::new(value)) {
|
|
172
|
+
PENDING_RELEASES.fetch_sub(1, Ordering::Relaxed);
|
|
173
|
+
drop(unsent);
|
|
174
|
+
}
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
drop(value)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
static RELEASE_THREAD: OnceLock<mpsc::Sender<Box<dyn Send>>> = OnceLock::new();
|
|
181
|
+
|
|
182
|
+
/// Starts the thread that frees released documents while no worker pool
|
|
183
|
+
/// exists, so freeing large trees does not stall JavaScript. Documents large
|
|
184
|
+
/// enough to be released later start it when they are created, because
|
|
185
|
+
/// finalizers must not create threads.
|
|
186
|
+
/// Trees with at least this many nodes return the parse's freed memory to
|
|
187
|
+
/// the system once built.
|
|
188
|
+
#[cfg(target_os = "linux")]
|
|
189
|
+
const RETURN_PAGES_NODES: usize = 100_000;
|
|
190
|
+
|
|
191
|
+
#[cfg(target_os = "linux")]
|
|
192
|
+
extern "C" {
|
|
193
|
+
fn mi_collect(force: bool);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/// Building a large tree grows and then frees big buffers, such as a node
|
|
197
|
+
/// vector that doubled several times. mimalloc keeps freed pages resident for
|
|
198
|
+
/// reuse; after a large parse, return them on the parsing thread instead.
|
|
199
|
+
pub(crate) fn return_parse_pages(nodes: usize) {
|
|
200
|
+
#[cfg(target_os = "linux")]
|
|
201
|
+
if nodes >= RETURN_PAGES_NODES {
|
|
202
|
+
// SAFETY: mimalloc's own collection frees only memory no block uses.
|
|
203
|
+
unsafe { mi_collect(true) };
|
|
204
|
+
}
|
|
205
|
+
#[cfg(not(target_os = "linux"))]
|
|
206
|
+
let _ = nodes;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
pub(crate) fn start_release_thread() {
|
|
210
|
+
RELEASE_THREAD.get_or_init(|| {
|
|
211
|
+
let (sender, receiver) = mpsc::channel::<Box<dyn Send>>();
|
|
212
|
+
// A thread that cannot start drops its receiver; sends then fail and
|
|
213
|
+
// values drop on the caller.
|
|
214
|
+
let _ = std::thread::Builder::new()
|
|
215
|
+
.name("cheerox-release".into())
|
|
216
|
+
.spawn(move || {
|
|
217
|
+
for value in receiver {
|
|
218
|
+
drop(value);
|
|
219
|
+
PENDING_RELEASES.fetch_sub(1, Ordering::Relaxed);
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
sender
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/// Permits for up to `wanted` workers that are free now, without waiting.
|
|
227
|
+
fn available_permits(wanted: usize) -> Vec<Permit> {
|
|
228
|
+
let shared = runtime();
|
|
229
|
+
let limit = {
|
|
230
|
+
let config = shared.configuration.lock().unwrap();
|
|
231
|
+
if config.error.is_some() {
|
|
232
|
+
return Vec::new();
|
|
233
|
+
}
|
|
234
|
+
config.threads as u32
|
|
235
|
+
};
|
|
236
|
+
let mut used = shared.used.lock().unwrap();
|
|
237
|
+
let count = (limit.saturating_sub(*used) as usize).min(wanted);
|
|
238
|
+
*used += count as u32;
|
|
239
|
+
shared.peak.fetch_max(*used, Ordering::Relaxed);
|
|
240
|
+
(0..count).map(|_| Permit).collect()
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/// Maps chunks of `0..count` in order. It uses only the shared pool's workers
|
|
244
|
+
/// that are idle now, so work on the JavaScript thread never waits on other
|
|
245
|
+
/// batches; with none idle, or little work, it runs on the calling thread.
|
|
246
|
+
pub(crate) fn chunked_map<U: Send>(
|
|
247
|
+
count: usize,
|
|
248
|
+
chunk: usize,
|
|
249
|
+
operation: impl Fn(std::ops::Range<usize>) -> Vec<U> + Sync + Send,
|
|
250
|
+
) -> Vec<U> {
|
|
251
|
+
// A pool worker already holds its batch's permit; waiting there on jobs
|
|
252
|
+
// installed behind other batches' work could leave every worker waiting.
|
|
253
|
+
if rayon::current_thread_index().is_some() {
|
|
254
|
+
return operation(0..count);
|
|
255
|
+
}
|
|
256
|
+
let chunks = count.div_ceil(chunk.max(1));
|
|
257
|
+
let permits = if chunks > 1 {
|
|
258
|
+
available_permits(chunks)
|
|
259
|
+
} else {
|
|
260
|
+
Vec::new()
|
|
261
|
+
};
|
|
262
|
+
let pool = (permits.len() > 1).then(internal_pool).and_then(Result::ok);
|
|
263
|
+
let Some(pool) = pool else {
|
|
264
|
+
drop(permits);
|
|
265
|
+
return operation(0..count);
|
|
266
|
+
};
|
|
267
|
+
let _batch = batch();
|
|
268
|
+
let size = count.div_ceil(permits.len());
|
|
269
|
+
let ranges: Vec<_> = (0..count)
|
|
270
|
+
.step_by(size)
|
|
271
|
+
.map(|start| start..(start + size).min(count))
|
|
272
|
+
.collect();
|
|
273
|
+
let results: Vec<Vec<U>> = pool.install(|| ranges.into_par_iter().map(&operation).collect());
|
|
274
|
+
drop(permits);
|
|
275
|
+
let mut merged = Vec::with_capacity(results.iter().map(Vec::len).sum());
|
|
276
|
+
for chunk in results {
|
|
277
|
+
merged.extend(chunk);
|
|
278
|
+
}
|
|
279
|
+
merged
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
pub(crate) struct Batch;
|
|
283
|
+
pub(crate) fn batch() -> Batch {
|
|
284
|
+
runtime().batches.fetch_add(1, Ordering::Relaxed);
|
|
285
|
+
Batch
|
|
286
|
+
}
|
|
287
|
+
impl Drop for Batch {
|
|
288
|
+
fn drop(&mut self) {
|
|
289
|
+
runtime().batches.fetch_sub(1, Ordering::Relaxed);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/// Maps `items` in order, on at most `threads` threads within the
|
|
294
|
+
/// process-wide budget. The calling thread waits for one permit and maps
|
|
295
|
+
/// items itself; pool workers help only with permits that are free now.
|
|
296
|
+
/// Pool jobs therefore never wait for permits, and a free permit always has
|
|
297
|
+
/// an idle worker: a worker that waited could hold the thread a permit's
|
|
298
|
+
/// holder needs in order to finish.
|
|
299
|
+
pub(crate) fn parallel_map<T: Sync, U: Send>(
|
|
300
|
+
items: &[T],
|
|
301
|
+
threads: usize,
|
|
302
|
+
operation: impl Fn(&T) -> Result<U> + Sync + Send,
|
|
303
|
+
) -> Result<Vec<U>> {
|
|
304
|
+
let _batch = batch();
|
|
305
|
+
let results: Vec<Mutex<Option<Result<U>>>> = items.iter().map(|_| Mutex::new(None)).collect();
|
|
306
|
+
let next = AtomicUsize::new(0);
|
|
307
|
+
let map_remaining = || loop {
|
|
308
|
+
let index = next.fetch_add(1, Ordering::Relaxed);
|
|
309
|
+
let Some(item) = items.get(index) else {
|
|
310
|
+
break;
|
|
311
|
+
};
|
|
312
|
+
*results[index].lock().unwrap() = Some(operation(item));
|
|
313
|
+
};
|
|
314
|
+
// A pool worker already holds the permit of the job it runs.
|
|
315
|
+
let own = if rayon::current_thread_index().is_some() {
|
|
316
|
+
None
|
|
317
|
+
} else {
|
|
318
|
+
Some(permit()?)
|
|
319
|
+
};
|
|
320
|
+
let wanted_helpers = threads.max(1).min(items.len()).saturating_sub(1);
|
|
321
|
+
let helpers = if own.is_some() && wanted_helpers > 0 {
|
|
322
|
+
available_permits(wanted_helpers)
|
|
323
|
+
} else {
|
|
324
|
+
Vec::new()
|
|
325
|
+
};
|
|
326
|
+
if helpers.is_empty() {
|
|
327
|
+
map_remaining();
|
|
328
|
+
} else {
|
|
329
|
+
let pool = shared_pool()?;
|
|
330
|
+
pool.in_place_scope(|scope| {
|
|
331
|
+
for helper in helpers {
|
|
332
|
+
scope.spawn(|_| {
|
|
333
|
+
map_remaining();
|
|
334
|
+
drop(helper);
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
map_remaining();
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
drop(own);
|
|
341
|
+
results
|
|
342
|
+
.into_iter()
|
|
343
|
+
.map(|result| result.into_inner().unwrap().expect("every item ran"))
|
|
344
|
+
.collect()
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
#[cfg(test)]
|
|
348
|
+
mod tests {
|
|
349
|
+
use super::*;
|
|
350
|
+
#[test]
|
|
351
|
+
fn shared_budget_and_order_survive_concurrent_batches() {
|
|
352
|
+
let budget = worker_pool_info().threads;
|
|
353
|
+
let inputs: Vec<_> = (0..32).collect();
|
|
354
|
+
std::thread::scope(|scope| {
|
|
355
|
+
let handles: Vec<_> = (0..4)
|
|
356
|
+
.map(|_| scope.spawn(|| parallel_map(&inputs, 2, |value| Ok(value * 2)).unwrap()))
|
|
357
|
+
.collect();
|
|
358
|
+
for handle in handles {
|
|
359
|
+
assert_eq!(
|
|
360
|
+
handle.join().unwrap(),
|
|
361
|
+
inputs.iter().map(|value| value * 2).collect::<Vec<_>>()
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
let info = worker_pool_info();
|
|
366
|
+
assert!(info.peak_active_workers <= budget);
|
|
367
|
+
}
|
|
368
|
+
}
|