makiri 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/libfuzzer.yml +1 -1
- data/.github/workflows/verify.yml +37 -0
- data/CHANGELOG.md +19 -4
- data/README.md +16 -0
- data/Rakefile +13 -0
- data/ext/makiri/bridge/bridge.h +28 -4
- data/ext/makiri/bridge/ruby_string.c +25 -0
- data/ext/makiri/core/mkr_alloc.c +6 -0
- data/ext/makiri/core/mkr_alloc.h +13 -4
- data/ext/makiri/core/mkr_buf.h +1 -0
- data/ext/makiri/core/mkr_text.h +28 -9
- data/ext/makiri/core/mkr_utf8.h +8 -1
- data/ext/makiri/fuzz/Makefile +14 -6
- data/ext/makiri/fuzz/xml_xpath_fuzz.c +87 -0
- data/ext/makiri/glue/ruby_html_css.c +99 -10
- data/ext/makiri/glue/ruby_html_mutate.c +5 -5
- data/ext/makiri/xpath/mkr_xpath.c +12 -0
- data/ext/makiri/xpath/mkr_xpath_eval_body.h +18 -8
- data/ext/makiri/xpath/mkr_xpath_funcs_body.h +47 -38
- data/ext/makiri/xpath/mkr_xpath_xml_selftest.c +7 -10
- data/lib/makiri/version.rb +1 -1
- data/script/check_c_safety.rb +11 -0
- data/script/check_c_safety_allowlist.yml +9 -12
- data/suppressions/ruby.supp +22 -0
- data/verify/Makefile +237 -0
- data/verify/cbmc_models.c +38 -0
- data/verify/harness_alloc.c +257 -0
- data/verify/harness_buf.c +167 -0
- data/verify/harness_hash.c +28 -0
- data/verify/harness_span.c +213 -0
- data/verify/harness_spanbuf.c +98 -0
- data/verify/harness_utf8.c +70 -0
- data/verify/harness_utf8_chain.c +64 -0
- data/verify/harness_utf8_words.c +54 -0
- data/verify/harness_xml_chars.c +59 -0
- data/verify/harness_xml_expand.c +72 -0
- data/verify/harness_xml_parse.c +42 -0
- data/verify/harness_xpath_lex.c +51 -0
- data/verify/harness_xpath_number.c +68 -0
- data/verify/harness_xpath_parse.c +57 -0
- data/verify/selftest_main.c +16 -0
- data/verify/stub.c +10 -0
- data/verify/verify.h +54 -0
- metadata +22 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ad4966a3bf44b4cbc01822bab034227c3c5a5c18347135bfc5cfbc13963a42eb
|
|
4
|
+
data.tar.gz: 4669d45c028deb66343788e8adb27c72819fb78c71a3f6efa14f8c5acc07efc4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a076b9c0c2f8e790d6bf433912f8f27cf1c9439de66dbcd403a010736ee26e3716531c975969ea03274bb0fdac34e4c46ab25dbc6777eb50ec753e41a0c95e90
|
|
7
|
+
data.tar.gz: ad0d15f5fc4b6de3bf8850b21f69b0e66848c0dfe9f19b1304e9f0acde2754979506ca618a2af0449c9191052491fddc3b9ace9f226beed4b94349c48b4c09cc
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
name: Verify
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, master]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
cbmc:
|
|
11
|
+
name: CBMC proofs
|
|
12
|
+
runs-on: ubuntu-24.04
|
|
13
|
+
timeout-minutes: 20
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- name: Checkout (with vendored Lexbor submodule)
|
|
17
|
+
uses: actions/checkout@v6
|
|
18
|
+
with:
|
|
19
|
+
submodules: recursive
|
|
20
|
+
|
|
21
|
+
- name: Set up Ruby
|
|
22
|
+
uses: ruby/setup-ruby@v1
|
|
23
|
+
with:
|
|
24
|
+
ruby-version: "3.4"
|
|
25
|
+
bundler-cache: true
|
|
26
|
+
|
|
27
|
+
- name: Install CBMC
|
|
28
|
+
env:
|
|
29
|
+
CBMC_VERSION: "6.9.0"
|
|
30
|
+
run: |
|
|
31
|
+
curl -fsSLO "https://github.com/diffblue/cbmc/releases/download/cbmc-${CBMC_VERSION}/ubuntu-24.04-cbmc-${CBMC_VERSION}-Linux.deb"
|
|
32
|
+
sudo apt-get update
|
|
33
|
+
sudo apt-get install -y "./ubuntu-24.04-cbmc-${CBMC_VERSION}-Linux.deb"
|
|
34
|
+
cbmc --version
|
|
35
|
+
|
|
36
|
+
- name: Run verification suite
|
|
37
|
+
run: bundle exec rake verify
|
data/CHANGELOG.md
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## [0.6.0] - 2026-07-05
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
### Changed
|
|
6
|
+
|
|
7
|
+
* Text and comment node content (`create_text_node`, `create_comment`,
|
|
8
|
+
`content=`) and attribute values (`[]=`, `set_attribute_ns`) now accept an
|
|
9
|
+
embedded NUL (U+0000) and store it verbatim, matching the WHATWG DOM and
|
|
10
|
+
browsers, instead of raising. Names, tag names, namespaces, CSS selectors, and
|
|
11
|
+
XPath expressions stay NUL-strict, and invalid UTF-8 is still rejected. XML
|
|
12
|
+
documents (`Makiri::XML`) are unchanged and still reject NUL everywhere, since
|
|
13
|
+
XML 1.0 has no legal U+0000 character.
|
|
14
|
+
|
|
15
|
+
## [0.5.1] - 2026-06-22
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
* Faster CSS queries that reuse the same selector: compiled selectors are now
|
|
20
|
+
cached and reused across queries instead of being re-parsed each time.
|
|
7
21
|
|
|
8
22
|
## [0.5.0] - 2026-06-14
|
|
9
23
|
|
|
@@ -362,7 +376,8 @@ libxml2 / libxslt dependency at any layer**.
|
|
|
362
376
|
domxpath, CSS differential vs `Nokogiri::HTML5`). GitHub Actions CI across
|
|
363
377
|
Ruby 3.2–4.0 × Ubuntu/macOS plus a sanitizer job.
|
|
364
378
|
|
|
365
|
-
[Unreleased]: https://github.com/takahashim/makiri/compare/v0.5.
|
|
379
|
+
[Unreleased]: https://github.com/takahashim/makiri/compare/v0.5.1...HEAD
|
|
380
|
+
[0.5.1]: https://github.com/takahashim/makiri/compare/v0.5.0...v0.5.1
|
|
366
381
|
[0.5.0]: https://github.com/takahashim/makiri/compare/v0.4.0...v0.5.0
|
|
367
382
|
[0.4.0]: https://github.com/takahashim/makiri/compare/v0.3.0...v0.4.0
|
|
368
383
|
[0.3.0]: https://github.com/takahashim/makiri/compare/v0.2.0...v0.3.0
|
data/README.md
CHANGED
|
@@ -267,6 +267,22 @@ Detailed, test-backed notes live in `spec/conformance/README.md`.
|
|
|
267
267
|
* Type selectors are ASCII case-insensitive (CSS-correct for HTML; `LI` matches `<li>`)
|
|
268
268
|
* `Nokogiri::HTML5` is case-sensitive there.
|
|
269
269
|
|
|
270
|
+
### Text input (mutation APIs)
|
|
271
|
+
|
|
272
|
+
* Programmatic string arguments must be **valid UTF-8** (invalid bytes raise
|
|
273
|
+
`Makiri::Error`, never silently repaired - unlike HTML *parsing*, which decodes
|
|
274
|
+
leniently to U+FFFD).
|
|
275
|
+
* An embedded **NUL (U+0000)** is **accepted in HTML data-family content** -
|
|
276
|
+
text/comment node content (`create_text_node`, `create_comment`, `content=`)
|
|
277
|
+
and attribute values (`[]=`, `set_attribute_ns`) - and stored/read back
|
|
278
|
+
verbatim, matching the WHATWG DOM / browsers (`document.createTextNode("\0")`).
|
|
279
|
+
It is still **rejected** in names, tag names, namespaces, PI target/data, CSS
|
|
280
|
+
selectors, and XPath expressions/variable names (a NUL there raises).
|
|
281
|
+
* On re-parse, the HTML tokenizer replaces a U+0000 in text/attributes with
|
|
282
|
+
U+FFFD (WHATWG), so a serialized-then-reparsed round-trip is not byte-identical.
|
|
283
|
+
* `Makiri::XML` rejects NUL everywhere: XML 1.0 has no legal U+0000 character,
|
|
284
|
+
so admitting it would produce non-well-formed XML.
|
|
285
|
+
|
|
270
286
|
## Conformance
|
|
271
287
|
|
|
272
288
|
The XPath engine and XML parser are original code, so their correctness is held by
|
data/Rakefile
CHANGED
|
@@ -287,6 +287,12 @@ task :oom do
|
|
|
287
287
|
puts "(injection build left in place; run `rake clean compile` to restore a normal build)"
|
|
288
288
|
end
|
|
289
289
|
|
|
290
|
+
desc "CBMC proofs over the Ruby/Lexbor-free carve-out (core + XML + XPath front; " \
|
|
291
|
+
"needs cbmc; see docs/formal_verification.ja.md and verify/Makefile)"
|
|
292
|
+
task :verify do
|
|
293
|
+
sh "make -C verify smoke selftest cbmc"
|
|
294
|
+
end
|
|
295
|
+
|
|
290
296
|
desc "Run the performance benchmark (Makiri vs Nokogiri reference)"
|
|
291
297
|
task bench: :compile do
|
|
292
298
|
# Run outside the bundle so the bench-only gems (nokogiri, benchmark-ips)
|
|
@@ -303,6 +309,13 @@ task "bench:xml" => :compile do
|
|
|
303
309
|
end
|
|
304
310
|
end
|
|
305
311
|
|
|
312
|
+
desc "Run the CSS selector-query benchmark on a note.com-style SPA page (set BENCH_CARDS)"
|
|
313
|
+
task "bench:css" => :compile do
|
|
314
|
+
Bundler.with_unbundled_env do
|
|
315
|
+
sh "#{FileUtils::RUBY} -Ilib bench/bench_css_query.rb"
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
306
319
|
namespace :conformance do
|
|
307
320
|
desc "WHATWG HTML5 parsing conformance: run html5lib-tests through Makiri"
|
|
308
321
|
task html5: :compile do
|
data/ext/makiri/bridge/bridge.h
CHANGED
|
@@ -24,6 +24,22 @@ typedef struct {
|
|
|
24
24
|
size_t len;
|
|
25
25
|
} mkr_ruby_borrowed_text_t;
|
|
26
26
|
|
|
27
|
+
/* A validated, borrowed view of a Ruby String argument for a DATA-FAMILY
|
|
28
|
+
* mutation (text/comment/CDATA node content, attribute value). Contract: valid
|
|
29
|
+
* UTF-8, interior NUL ALLOWED, NUL-terminated at ptr[len] but - unlike
|
|
30
|
+
* mkr_ruby_borrowed_text_t - NOT usable as a C string, because an interior NUL
|
|
31
|
+
* may truncate it. Every consumer MUST pass (ptr, len) explicitly. `value`
|
|
32
|
+
* keeps `ptr` alive while the view stays on the C stack. This type is
|
|
33
|
+
* deliberately distinct from mkr_ruby_borrowed_text_t so the compiler forbids a
|
|
34
|
+
* NUL-bearing value from reaching a name or engine API that requires the
|
|
35
|
+
* NUL-free contract (DOM createTextNode/setAttribute must accept U+0000; names
|
|
36
|
+
* and engine strings still must not). */
|
|
37
|
+
typedef struct {
|
|
38
|
+
VALUE value;
|
|
39
|
+
const char *ptr;
|
|
40
|
+
size_t len;
|
|
41
|
+
} mkr_ruby_borrowed_data_t;
|
|
42
|
+
|
|
27
43
|
/* A borrowed raw byte view of a Ruby String. This deliberately does NOT enforce
|
|
28
44
|
* Makiri's strict text contract; HTML parsing consumes raw bytes and decodes
|
|
29
45
|
* invalid UTF-8 leniently. */
|
|
@@ -37,6 +53,12 @@ typedef struct {
|
|
|
37
53
|
* Makiri::Error (naming +what+) on a NUL byte or invalid UTF-8. */
|
|
38
54
|
mkr_ruby_borrowed_text_t mkr_ruby_verified_text(VALUE in, const char *what);
|
|
39
55
|
|
|
56
|
+
/* Constructor for mkr_ruby_borrowed_data_t (see its contract above): coerce +in+
|
|
57
|
+
* to a String, requiring valid UTF-8 but permitting an interior NUL. Raises
|
|
58
|
+
* Makiri::Error (naming +what+) on invalid UTF-8 only. For data-family values;
|
|
59
|
+
* names and engine strings use mkr_ruby_verified_text. */
|
|
60
|
+
mkr_ruby_borrowed_data_t mkr_ruby_verified_data(VALUE in, const char *what);
|
|
61
|
+
|
|
40
62
|
/* Coerce +in+ to a String and return its raw bytes. No UTF-8 / NUL validation.
|
|
41
63
|
* The returned `ptr` borrows the bytes of `value`; use mkr_ruby_copy_bytes when
|
|
42
64
|
* the buffer must outlive the source String. */
|
|
@@ -84,10 +106,12 @@ const char *mkr_ruby_try_verified_text(VALUE sv, size_t max_bytes,
|
|
|
84
106
|
* escape. Embedded NULs are intentionally truncated by %s formatting. */
|
|
85
107
|
void mkr_ruby_exception_message(VALUE exc, char *buf, size_t len);
|
|
86
108
|
|
|
87
|
-
/* The
|
|
88
|
-
* (from mkr_ruby_verified_text or mkr_ruby_try_verified_text) into the token
|
|
89
|
-
* XPath engine accepts. The returned token borrows v.ptr; the caller must
|
|
90
|
-
* v.value alive for the duration of the engine call.
|
|
109
|
+
/* The Ruby-bound constructor of mkr_verified_text_t. Turns an already-validated
|
|
110
|
+
* view (from mkr_ruby_verified_text or mkr_ruby_try_verified_text) into the token
|
|
111
|
+
* the XPath engine accepts. The returned token borrows v.ptr; the caller must
|
|
112
|
+
* keep v.value alive for the duration of the engine call. The sole Ruby-free
|
|
113
|
+
* constructor is mkr_verified_text_lit (core/mkr_text.h), for compile-time
|
|
114
|
+
* literals. */
|
|
91
115
|
mkr_verified_text_t mkr_verified_text_from_view(mkr_ruby_borrowed_text_t v);
|
|
92
116
|
|
|
93
117
|
/* Assemble a UTF-8 Ruby String of exactly +total+ bytes by concatenating +n+
|
|
@@ -115,6 +115,31 @@ mkr_ruby_verified_text(VALUE in, const char *what)
|
|
|
115
115
|
return v;
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
+
mkr_ruby_borrowed_data_t
|
|
119
|
+
mkr_ruby_verified_data(VALUE in, const char *what)
|
|
120
|
+
{
|
|
121
|
+
VALUE s = rb_String(in);
|
|
122
|
+
const char *ptr = RSTRING_PTR(s);
|
|
123
|
+
size_t len = (size_t)RSTRING_LEN(s);
|
|
124
|
+
|
|
125
|
+
/* Data-family (text/comment/CDATA content, attribute value): only invalid
|
|
126
|
+
* UTF-8 is fatal; an interior NUL is permitted so DOM data can hold U+0000.
|
|
127
|
+
* mkr_verify_text is not reused because it raises on NUL. mkr_text_check is
|
|
128
|
+
* allocation-free, so the borrow above is not held across a GC point. */
|
|
129
|
+
switch (mkr_text_check(s, ptr, len)) {
|
|
130
|
+
case MKR_TEXT_INVALID_UTF8:
|
|
131
|
+
rb_raise(mkr_eError, "%s must be valid UTF-8", what);
|
|
132
|
+
case MKR_TEXT_HAS_NUL: /* permitted for data-family */
|
|
133
|
+
case MKR_TEXT_OK:
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
mkr_ruby_borrowed_data_t v;
|
|
137
|
+
v.value = s;
|
|
138
|
+
v.ptr = ptr;
|
|
139
|
+
v.len = len;
|
|
140
|
+
return v;
|
|
141
|
+
}
|
|
142
|
+
|
|
118
143
|
mkr_ruby_borrowed_bytes_t
|
|
119
144
|
mkr_ruby_bytes_view(VALUE in)
|
|
120
145
|
{
|
data/ext/makiri/core/mkr_alloc.c
CHANGED
|
@@ -39,6 +39,12 @@ mkr_reallocarray(void *ptr, size_t count, size_t elem)
|
|
|
39
39
|
free(ptr);
|
|
40
40
|
return NULL;
|
|
41
41
|
}
|
|
42
|
+
if (elem == 0) {
|
|
43
|
+
/* Fail closed like mkr_callocarray rather than fall through to a
|
|
44
|
+
* realloc(ptr, 0) whose free-or-not is implementation-defined; the
|
|
45
|
+
* caller keeps ownership of ptr. */
|
|
46
|
+
return NULL;
|
|
47
|
+
}
|
|
42
48
|
size_t bytes;
|
|
43
49
|
if (!mkr_size_mul(count, elem, &bytes)) {
|
|
44
50
|
return NULL; /* overflow: leave ptr unchanged */
|
data/ext/makiri/core/mkr_alloc.h
CHANGED
|
@@ -48,8 +48,10 @@ mkr_size_mul(size_t a, size_t b, size_t *out)
|
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
/* Compute a new capacity (in elements) that is >= need and grows geometrically
|
|
51
|
-
* from cap, with
|
|
52
|
-
*
|
|
51
|
+
* from cap, with new_cap*elem guaranteed to fit in size_t. Succeeds EXACTLY
|
|
52
|
+
* when need*elem fits: if the geometric target overshoots what elem allows, it
|
|
53
|
+
* falls back to `need` rather than failing, so the only failure is `need`
|
|
54
|
+
* itself being too large. (cbmc-alloc proves success <=> need*elem fits.) */
|
|
53
55
|
static inline bool
|
|
54
56
|
mkr_grow_capacity(size_t cap, size_t need, size_t elem, size_t *new_cap)
|
|
55
57
|
{
|
|
@@ -59,14 +61,21 @@ mkr_grow_capacity(size_t cap, size_t need, size_t elem, size_t *new_cap)
|
|
|
59
61
|
nc *= 2;
|
|
60
62
|
}
|
|
61
63
|
size_t bytes;
|
|
62
|
-
if (!mkr_size_mul(nc, elem, &bytes))
|
|
64
|
+
if (!mkr_size_mul(nc, elem, &bytes)) {
|
|
65
|
+
/* Geometric overshoot doesn't fit; retry at exactly `need` so a
|
|
66
|
+
* representable need never fails from the *2 headroom alone. */
|
|
67
|
+
nc = need;
|
|
68
|
+
if (!mkr_size_mul(nc, elem, &bytes)) return false;
|
|
69
|
+
}
|
|
63
70
|
*new_cap = nc;
|
|
64
71
|
return true;
|
|
65
72
|
}
|
|
66
73
|
|
|
67
74
|
/* realloc(ptr, count * elem) with the multiply overflow-checked. Returns NULL
|
|
68
75
|
* on overflow or OOM; the original ptr is then unchanged (caller still owns
|
|
69
|
-
* it). count == 0 frees and returns NULL
|
|
76
|
+
* it). count == 0 frees ptr and returns NULL; elem == 0 (with count > 0)
|
|
77
|
+
* returns NULL and leaves ptr unchanged (no implementation-defined
|
|
78
|
+
* realloc(ptr, 0)). */
|
|
70
79
|
void *mkr_reallocarray(void *ptr, size_t count, size_t elem);
|
|
71
80
|
|
|
72
81
|
/* calloc(count, elem) with the multiply overflow-checked, result zeroed.
|
data/ext/makiri/core/mkr_buf.h
CHANGED
|
@@ -143,6 +143,7 @@ static inline void
|
|
|
143
143
|
mkr_spanbuf_write(mkr_spanbuf_t *b, const void *src, size_t n)
|
|
144
144
|
{
|
|
145
145
|
if (!b->ok || n == 0) return;
|
|
146
|
+
if (src == NULL) { b->ok = false; return; } /* fail closed, never memcpy(,NULL,n) */
|
|
146
147
|
if (b->pos > b->cap || n > b->cap - b->pos) { b->ok = false; return; }
|
|
147
148
|
memcpy(b->buf + b->pos, src, n);
|
|
148
149
|
b->pos += n;
|
data/ext/makiri/core/mkr_text.h
CHANGED
|
@@ -22,17 +22,28 @@ extern "C" {
|
|
|
22
22
|
/* A borrowed byte slice whose contents are guaranteed to satisfy Makiri's
|
|
23
23
|
* engine text contract: valid UTF-8, no interior NUL, and NUL-terminated at
|
|
24
24
|
* ptr[len]. The XPath engine's public string entry points accept ONLY this
|
|
25
|
-
* type, so a raw `const char *` cannot be passed without a compile error.
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
25
|
+
* type, so a raw `const char *` cannot be passed without a compile error.
|
|
26
|
+
* Constructors:
|
|
27
|
+
* - mkr_verified_text_from_view() at the Ruby boundary (bridge), fed a view
|
|
28
|
+
* that mkr_ruby_verified_text() already validated;
|
|
29
|
+
* - mkr_verified_text_lit() below, for compile-time string literals (which
|
|
30
|
+
* meet the contract by construction).
|
|
31
|
+
* There is deliberately no Ruby-free constructor over runtime bytes, so
|
|
32
|
+
* unvalidated bytes have no path into the engine. (ptr is non-owning: the
|
|
33
|
+
* caller keeps it alive for the duration of the engine call.) */
|
|
31
34
|
typedef struct {
|
|
32
35
|
const char *ptr;
|
|
33
36
|
size_t len;
|
|
34
37
|
} mkr_verified_text_t;
|
|
35
38
|
|
|
39
|
+
/* Wrap a compile-time string literal as a verified-text token. The literal is
|
|
40
|
+
* NUL-terminated, NUL-free, and valid UTF-8 by construction, so it meets the
|
|
41
|
+
* engine text contract without validation - the sole Ruby-free mint, for
|
|
42
|
+
* selftest / fuzz harness literals that the bridge cannot reach. Do not pass a
|
|
43
|
+
* char* variable: sizeof(pointer) would not be the byte length. */
|
|
44
|
+
#define mkr_verified_text_lit(lit) \
|
|
45
|
+
((mkr_verified_text_t){ (lit), sizeof(lit) - 1 })
|
|
46
|
+
|
|
36
47
|
/* ---------------------------------------------------------------- */
|
|
37
48
|
/* string-type lattice */
|
|
38
49
|
/* ---------------------------------------------------------------- */
|
|
@@ -59,10 +70,18 @@ typedef struct {
|
|
|
59
70
|
* raw slice today. When something does, name it mkr_borrowed_bytes_t per the
|
|
60
71
|
* convention above; do not resurrect a placeholder before there is a user.
|
|
61
72
|
*
|
|
73
|
+
* One bridge-only variant sits BETWEEN bytes and text on the contract axis:
|
|
74
|
+
* mkr_ruby_borrowed_data_t (bridge.h) is "valid UTF-8 but interior NUL
|
|
75
|
+
* PERMITTED" - a strict superset of text - for DOM data-family mutations
|
|
76
|
+
* (text/comment/CDATA content, attribute value), where U+0000 is legal data.
|
|
77
|
+
* It never enters the engine (the engine still requires NUL-free text), so it
|
|
78
|
+
* is deliberately absent from this engine lattice.
|
|
79
|
+
*
|
|
62
80
|
* (*) mkr_verified_text_t (defined above) is the one deliberate naming exception:
|
|
63
|
-
* it is a borrowed valid text AND a capability token. Its
|
|
64
|
-
* mkr_verified_text_from_view() at the Ruby boundary
|
|
65
|
-
*
|
|
81
|
+
* it is a borrowed valid text AND a capability token. Its constructors are
|
|
82
|
+
* mkr_verified_text_from_view() at the Ruby boundary and mkr_verified_text_lit()
|
|
83
|
+
* for compile-time literals, so an unvalidated runtime const char* cannot
|
|
84
|
+
* reach the engine's public API. Internally the engine carries the
|
|
66
85
|
* freely-constructible mkr_borrowed_text_t instead.
|
|
67
86
|
*
|
|
68
87
|
* Conversions - the only sanctioned edges. The points that actually VALIDATE
|
data/ext/makiri/core/mkr_utf8.h
CHANGED
|
@@ -32,6 +32,10 @@
|
|
|
32
32
|
extern "C" {
|
|
33
33
|
#endif
|
|
34
34
|
|
|
35
|
+
/* src must be non-NULL when len > 0 (it is dereferenced through [0, len)); src
|
|
36
|
+
* may be NULL only when len == 0 (trivially valid). len > 0 with src == NULL is
|
|
37
|
+
* a caller-contract violation, not a fail-closed case - this is a hot path
|
|
38
|
+
* guarded at the call site, not here. */
|
|
35
39
|
bool mkr_utf8_valid(const unsigned char *src, size_t len);
|
|
36
40
|
|
|
37
41
|
/* mkr_utf8_decode1 - decode ONE code point from [p, p+len), strictly: rejects
|
|
@@ -53,7 +57,10 @@ mkr_utf8_decode1_span(const mkr_span_t *s, uint32_t *cp)
|
|
|
53
57
|
/* mkr_utf8_count_chars - count Unicode code points in [ptr, ptr+len): every
|
|
54
58
|
* byte that is NOT a 0x80..0xBF continuation byte starts a new code point.
|
|
55
59
|
* Length-bounded (does not rely on a NUL terminator); ptr may be NULL when
|
|
56
|
-
* len == 0. Used where XPath measures string length / offsets in characters.
|
|
60
|
+
* len == 0. Used where XPath measures string length / offsets in characters.
|
|
61
|
+
* Semantics: on VALID UTF-8 this equals the code-point count (proved in
|
|
62
|
+
* cbmc-utf8-chain); on invalid bytes it is only a lead-byte-class scan (no
|
|
63
|
+
* decoding), so the strong count/boundary guarantees hold for valid text. */
|
|
57
64
|
static inline size_t
|
|
58
65
|
mkr_utf8_count_chars(const char *ptr, size_t len)
|
|
59
66
|
{
|
data/ext/makiri/fuzz/Makefile
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# ext/makiri/fuzz/Makefile - libFuzzer harness build
|
|
2
2
|
#
|
|
3
3
|
# Usage:
|
|
4
|
-
# make xml_fuzz
|
|
5
|
-
# make xpath_fuzz
|
|
6
|
-
# make
|
|
4
|
+
# make xml_fuzz # build the XML parser harness
|
|
5
|
+
# make xpath_fuzz # build the XPath compile+eval harness (fixed document)
|
|
6
|
+
# make xml_xpath_fuzz # build the combined harness (fuzzer-controlled document + expression)
|
|
7
|
+
# make clean # remove build artifacts
|
|
7
8
|
#
|
|
8
9
|
# Requires: clang with libFuzzer support (usually part of the clang distribution)
|
|
9
10
|
# and the vendored Lexbor static library already built (via `rake compile`).
|
|
@@ -64,9 +65,9 @@ CORE_OBJS = $(patsubst $(EXT_DIR)/%.c,$(BUILD_DIR)/%.o,$(CORE_SRCS))
|
|
|
64
65
|
XML_OBJS = $(patsubst $(EXT_DIR)/%.c,$(BUILD_DIR)/%.o,$(XML_SRCS))
|
|
65
66
|
XPATH_OBJS = $(patsubst $(EXT_DIR)/%.c,$(BUILD_DIR)/%.o,$(XPATH_SRCS))
|
|
66
67
|
|
|
67
|
-
.PHONY: all clean check-libfuzzer xml_fuzz xpath_fuzz
|
|
68
|
+
.PHONY: all clean check-libfuzzer xml_fuzz xpath_fuzz xml_xpath_fuzz
|
|
68
69
|
|
|
69
|
-
all: check-libfuzzer xml_fuzz xpath_fuzz
|
|
70
|
+
all: check-libfuzzer xml_fuzz xpath_fuzz xml_xpath_fuzz
|
|
70
71
|
|
|
71
72
|
check-libfuzzer:
|
|
72
73
|
@mkdir -p $(BUILD_DIR)
|
|
@@ -79,6 +80,9 @@ xml_fuzz: $(BUILD_DIR)/xml_fuzz.o $(CORE_OBJS) $(XML_OBJS)
|
|
|
79
80
|
xpath_fuzz: $(BUILD_DIR)/xpath_fuzz.o $(CORE_OBJS) $(XML_OBJS) $(XPATH_OBJS)
|
|
80
81
|
$(CXX) $(LDFLAGS) -o $@ $^ $(LEXBOR_DST)/lib/liblexbor_static.a
|
|
81
82
|
|
|
83
|
+
xml_xpath_fuzz: $(BUILD_DIR)/xml_xpath_fuzz.o $(CORE_OBJS) $(XML_OBJS) $(XPATH_OBJS)
|
|
84
|
+
$(CXX) $(LDFLAGS) -o $@ $^ $(LEXBOR_DST)/lib/liblexbor_static.a
|
|
85
|
+
|
|
82
86
|
$(BUILD_DIR)/%.o: $(EXT_DIR)/%.c
|
|
83
87
|
@mkdir -p $(dir $@)
|
|
84
88
|
$(CLANG) $(CFLAGS) -c -o $@ $<
|
|
@@ -91,5 +95,9 @@ $(BUILD_DIR)/xpath_fuzz.o: xpath_fuzz.c
|
|
|
91
95
|
@mkdir -p $(dir $@)
|
|
92
96
|
$(CLANG) $(CFLAGS) -c -o $@ $<
|
|
93
97
|
|
|
98
|
+
$(BUILD_DIR)/xml_xpath_fuzz.o: xml_xpath_fuzz.c
|
|
99
|
+
@mkdir -p $(dir $@)
|
|
100
|
+
$(CLANG) $(CFLAGS) -c -o $@ $<
|
|
101
|
+
|
|
94
102
|
clean:
|
|
95
|
-
rm -rf $(BUILD_DIR) xml_fuzz xpath_fuzz
|
|
103
|
+
rm -rf $(BUILD_DIR) xml_fuzz xpath_fuzz xml_xpath_fuzz
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/* xml_xpath_fuzz.c - libFuzzer harness where the fuzzer controls BOTH the
|
|
2
|
+
* document and the expression.
|
|
3
|
+
*
|
|
4
|
+
* Complements the other two targets: xml_fuzz covers the parser alone, and
|
|
5
|
+
* xpath_fuzz covers the engine over one fixed document shape - so evaluator
|
|
6
|
+
* paths that depend on document structure (namespace scopes, index buckets,
|
|
7
|
+
* axis edge cases) only ever see that shape. Here the input's bytes up to the
|
|
8
|
+
* first NUL are the XML document and the rest is the expression (both engine
|
|
9
|
+
* contracts forbid embedded NUL, so the separator costs no coverage).
|
|
10
|
+
*
|
|
11
|
+
* Contract violations (invalid UTF-8, non-XML chars) are filtered the same
|
|
12
|
+
* way the Ruby bridge does, so the harness only exercises in-contract inputs;
|
|
13
|
+
* the validators themselves run on every input and are fuzzed for free.
|
|
14
|
+
* Ruby-free; the XML arena self-poisons under ASan, so intra-arena overruns
|
|
15
|
+
* are caught. */
|
|
16
|
+
#include <stddef.h>
|
|
17
|
+
#include <stdint.h>
|
|
18
|
+
#include <string.h>
|
|
19
|
+
|
|
20
|
+
#include "core/mkr_core.h"
|
|
21
|
+
#include "xml/mkr_xml.h"
|
|
22
|
+
#include "xpath/mkr_xpath.h"
|
|
23
|
+
|
|
24
|
+
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
|
|
25
|
+
|
|
26
|
+
int
|
|
27
|
+
LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
|
|
28
|
+
{
|
|
29
|
+
const uint8_t *sep = memchr(data, 0, size);
|
|
30
|
+
if (sep == NULL) return 0;
|
|
31
|
+
|
|
32
|
+
const char *xml = (const char *)data;
|
|
33
|
+
size_t xml_len = (size_t)(sep - data);
|
|
34
|
+
const char *expr = (const char *)sep + 1;
|
|
35
|
+
size_t expr_len = size - xml_len - 1;
|
|
36
|
+
|
|
37
|
+
/* In-contract filter, mirroring the bridge's strict gate. */
|
|
38
|
+
if (!mkr_utf8_valid((const unsigned char *)xml, xml_len)) return 0;
|
|
39
|
+
if (mkr_xml_validate_chars(xml, (uint32_t)xml_len) != 0) return 0;
|
|
40
|
+
if (!mkr_utf8_valid((const unsigned char *)expr, expr_len)) return 0;
|
|
41
|
+
|
|
42
|
+
mkr_xml_status_t st = MKR_XML_OK;
|
|
43
|
+
mkr_xml_doc_t *doc = mkr_xml_parse(xml, xml_len, &st);
|
|
44
|
+
if (doc == NULL) return 0;
|
|
45
|
+
|
|
46
|
+
mkr_xpath_context_t *ctx = mkr_xpath_context_new(doc->doc_node, doc->doc_node);
|
|
47
|
+
if (ctx != NULL) {
|
|
48
|
+
mkr_xpath_set_engine_kind(ctx, 1); /* XML monomorphization */
|
|
49
|
+
|
|
50
|
+
/* Shrink the per-evaluate budgets so a single hostile input can't
|
|
51
|
+
* stall the fuzzer; overruns fail closed, which is the point. */
|
|
52
|
+
mkr_xpath_limits_t *L = mkr_ctx_limits(ctx);
|
|
53
|
+
L->max_eval_ops = 20000;
|
|
54
|
+
L->max_nodeset_size = 1024;
|
|
55
|
+
L->max_string_bytes = 4096;
|
|
56
|
+
|
|
57
|
+
mkr_xpath_register_ns(ctx, mkr_verified_text_lit("d"), mkr_verified_text_lit("urn:d"));
|
|
58
|
+
|
|
59
|
+
/* The expression is the bytes after the separator NUL. libFuzzer
|
|
60
|
+
* gives exactly `size` bytes with no trailing terminator, so copy
|
|
61
|
+
* into an owned, NUL-terminated heap buffer and mint over that
|
|
62
|
+
* (mirroring xpath_fuzz.c): this supplies the NUL at ptr[len] the
|
|
63
|
+
* lexer's strtod / "%.10s" error path rely on. UTF-8 validity was
|
|
64
|
+
* checked above; the copy preserves it. */
|
|
65
|
+
char *expr_copy = mkr_strndup(expr, expr_len);
|
|
66
|
+
if (expr_copy == NULL) {
|
|
67
|
+
mkr_xpath_context_free(ctx);
|
|
68
|
+
mkr_xml_doc_destroy(doc);
|
|
69
|
+
return 0;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
mkr_xpath_error_t err = {0};
|
|
73
|
+
mkr_verified_text_t e = { expr_copy, expr_len };
|
|
74
|
+
mkr_node_t *ast = mkr_parse(e, L, &err);
|
|
75
|
+
if (ast != NULL) {
|
|
76
|
+
mkr_xpath_value_t v = {0};
|
|
77
|
+
(void)mkr_xpath_eval_compiled(ctx, ast, &v, &err);
|
|
78
|
+
mkr_xpath_value_clear(&v);
|
|
79
|
+
mkr_node_free(ast);
|
|
80
|
+
}
|
|
81
|
+
mkr_xpath_error_clear(&err);
|
|
82
|
+
free(expr_copy);
|
|
83
|
+
mkr_xpath_context_free(ctx);
|
|
84
|
+
}
|
|
85
|
+
mkr_xml_doc_destroy(doc);
|
|
86
|
+
return 0;
|
|
87
|
+
}
|
|
@@ -33,6 +33,34 @@ static lxb_css_selectors_t *g_css_sel;
|
|
|
33
33
|
static lxb_selectors_t *g_selectors;
|
|
34
34
|
static int g_css_ready;
|
|
35
35
|
|
|
36
|
+
/* Compiled-selector cache: a Ruby Hash mapping a selector String to the parsed
|
|
37
|
+
* lxb_css_selector_list_t (stored as an Integer pointer). Parsing a selector is
|
|
38
|
+
* the dominant cost when the same selectors are queried over and over (a
|
|
39
|
+
* querySelector-heavy SPA fires tens of thousands of identical queries), so the
|
|
40
|
+
* compiled list is reused instead of re-parsed. The parsed lists live in the
|
|
41
|
+
* shared g_css_mem arena, which is therefore NOT cleaned per call (the original
|
|
42
|
+
* code reset it after every query); to bound growth, when the cache fills we drop
|
|
43
|
+
* every compiled list at once (clean the arena + clear the Hash) and start over. */
|
|
44
|
+
static VALUE g_css_cache;
|
|
45
|
+
#define MKR_CSS_CACHE_CAP 256
|
|
46
|
+
|
|
47
|
+
/* Adaptive caching. Holding many distinct compiled lists in the shared arena
|
|
48
|
+
* makes each new parse slower (a bigger arena = more allocator work), so a flood
|
|
49
|
+
* of one-off selectors -- e.g. getElementById on unique React `useId` ids, which
|
|
50
|
+
* are never requeried -- turned the cache into a net loss vs the original
|
|
51
|
+
* parse+clean (measured ~22% slower per call). Track the cache hit rate over a
|
|
52
|
+
* window and, when it is low, BYPASS the cache: parse + clean per call, exactly
|
|
53
|
+
* as before, so the arena stays small and the worst case is merely "as fast as no
|
|
54
|
+
* cache". Periodically drop back into caching to re-test, so a workload that
|
|
55
|
+
* starts repeating selectors regains the cache. */
|
|
56
|
+
static size_t g_css_win; /* lookups in the current window */
|
|
57
|
+
static size_t g_css_win_hits; /* cache hits in the current window */
|
|
58
|
+
static int g_css_bypass; /* 1 = parse+clean, no caching */
|
|
59
|
+
static size_t g_css_bypass_runs; /* consecutive bypass windows */
|
|
60
|
+
#define MKR_CSS_WIN 1024 /* re-evaluate the hit rate every N lookups */
|
|
61
|
+
#define MKR_CSS_MIN_HIT_PCT 15 /* below this hit rate, bypass the cache */
|
|
62
|
+
#define MKR_CSS_RETEST_GAP 32 /* re-test caching every N bypass windows */
|
|
63
|
+
|
|
36
64
|
/* Build the shared engine on first use; raises Makiri::Error on init failure
|
|
37
65
|
* (leaving the globals unset, so a later call retries). */
|
|
38
66
|
static void
|
|
@@ -179,22 +207,83 @@ mkr_with_compiled_selector(VALUE rb_selector, lxb_dom_node_t *node,
|
|
|
179
207
|
|
|
180
208
|
mkr_css_engine_init(); /* raises on init failure */
|
|
181
209
|
|
|
182
|
-
|
|
183
|
-
|
|
210
|
+
if (g_css_cache == 0) {
|
|
211
|
+
g_css_cache = rb_hash_new();
|
|
212
|
+
rb_gc_register_address(&g_css_cache);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/* Adaptive window: every MKR_CSS_WIN lookups, decide whether caching is
|
|
216
|
+
* paying off. A low hit rate (mostly one-off selectors) means the cache is
|
|
217
|
+
* only growing the arena and slowing parses, so bypass it; periodically
|
|
218
|
+
* re-test so a workload that becomes selector-repetitive regains the cache. */
|
|
219
|
+
if (++g_css_win > MKR_CSS_WIN) {
|
|
220
|
+
if (!g_css_bypass) {
|
|
221
|
+
if (g_css_win_hits * 100 < (size_t)MKR_CSS_WIN * MKR_CSS_MIN_HIT_PCT) {
|
|
222
|
+
g_css_bypass = 1;
|
|
223
|
+
g_css_bypass_runs = 0;
|
|
224
|
+
lxb_css_memory_clean(g_css_mem); /* drop the cached lists' arena */
|
|
225
|
+
rb_hash_clear(g_css_cache);
|
|
226
|
+
}
|
|
227
|
+
} else if (++g_css_bypass_runs >= MKR_CSS_RETEST_GAP) {
|
|
228
|
+
g_css_bypass = 0; /* re-test caching over the next window */
|
|
229
|
+
}
|
|
230
|
+
g_css_win = 1;
|
|
231
|
+
g_css_win_hits = 0;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (!g_css_bypass) {
|
|
235
|
+
/* Reuse the compiled selector when we have already parsed this string. */
|
|
236
|
+
VALUE cached = rb_hash_lookup(g_css_cache, sv.value);
|
|
237
|
+
if (!NIL_P(cached)) {
|
|
238
|
+
lxb_css_selector_list_t *list =
|
|
239
|
+
(lxb_css_selector_list_t *)(intptr_t)NUM2LL(cached);
|
|
240
|
+
g_css_win_hits++;
|
|
241
|
+
(void)run(g_selectors, node, list, u);
|
|
242
|
+
/* The traversal engine self-cleans; the cached list + arena persist. */
|
|
243
|
+
RB_GC_GUARD(sv.value);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/* Cache miss. Bound the cache before parsing: when full, drop every
|
|
248
|
+
* compiled list at once by cleaning the shared arena, then start fresh —
|
|
249
|
+
* so the new list is parsed into the now-empty arena (cleaning AFTER
|
|
250
|
+
* parsing would invalidate it). */
|
|
251
|
+
if (RHASH_SIZE(g_css_cache) >= MKR_CSS_CACHE_CAP) {
|
|
252
|
+
lxb_css_memory_clean(g_css_mem);
|
|
253
|
+
rb_hash_clear(g_css_cache);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
lxb_css_selector_list_t *list =
|
|
257
|
+
lxb_css_selectors_parse(g_css_parser, (const lxb_char_t *)sv.ptr, sv.len);
|
|
258
|
+
int syntax_error = (list == NULL || g_css_parser->status != LXB_STATUS_OK);
|
|
259
|
+
|
|
260
|
+
/* Return the parser to its CLEAN stage; do NOT clean the memory arena —
|
|
261
|
+
* the freshly parsed list lives there and we keep it. */
|
|
262
|
+
lxb_css_parser_clean(g_css_parser);
|
|
263
|
+
|
|
264
|
+
if (syntax_error) {
|
|
265
|
+
rb_raise(mkr_eCSSSyntaxError, "invalid CSS selector: %" PRIsVALUE, sv.value);
|
|
266
|
+
}
|
|
184
267
|
|
|
185
|
-
|
|
186
|
-
|
|
268
|
+
/* Cache the compiled list (the Hash dups + freezes the String key), then
|
|
269
|
+
* run. Store the pointer as a Ruby Integer; the list outlives the call. */
|
|
270
|
+
rb_hash_aset(g_css_cache, sv.value, LL2NUM((long long)(intptr_t)list));
|
|
187
271
|
(void)run(g_selectors, node, list, u);
|
|
272
|
+
RB_GC_GUARD(sv.value);
|
|
273
|
+
return;
|
|
188
274
|
}
|
|
189
275
|
|
|
190
|
-
/*
|
|
191
|
-
*
|
|
192
|
-
|
|
193
|
-
|
|
276
|
+
/* Bypass mode: parse + clean per call (the original behavior), so the arena
|
|
277
|
+
* stays small and a one-off-selector flood is no slower than no cache. */
|
|
278
|
+
lxb_css_selector_list_t *l0 =
|
|
279
|
+
lxb_css_selectors_parse(g_css_parser, (const lxb_char_t *)sv.ptr, sv.len);
|
|
280
|
+
int err0 = (l0 == NULL || g_css_parser->status != LXB_STATUS_OK);
|
|
281
|
+
if (!err0) {
|
|
282
|
+
(void)run(g_selectors, node, l0, u);
|
|
283
|
+
}
|
|
194
284
|
lxb_css_memory_clean(g_css_mem);
|
|
195
285
|
lxb_css_parser_clean(g_css_parser);
|
|
196
|
-
|
|
197
|
-
if (syntax_error) {
|
|
286
|
+
if (err0) {
|
|
198
287
|
rb_raise(mkr_eCSSSyntaxError, "invalid CSS selector: %" PRIsVALUE, sv.value);
|
|
199
288
|
}
|
|
200
289
|
RB_GC_GUARD(sv.value);
|