leptris 1.9.375.0 → 1.9.376.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fa27f3dd652c356d4b8b939f40aba24b30866302e3d4c3cfb7fa62c032de7415
4
- data.tar.gz: fe4921f2e63255893c97cb83bbfa845a2fe92c023c7b1a2a900794e183b30ba8
3
+ metadata.gz: a0399e31ae9c0ba48036d105e43bcbd52bfed938ae0c7f588d072e36ce7cd511
4
+ data.tar.gz: f9b7cfe41cec036205e5feeda04553a060168d780c699341d8b698b856da59c5
5
5
  SHA512:
6
- metadata.gz: e67786d718aa5a26dcb59e67cbd8144f0de8b4c0327ee64ec6a58d0ff89e2bd1512bb54e480e2c649252011361e77f6d811afdaf7ce6f070aa4d77a940fea7f9
7
- data.tar.gz: 879fe1f675823fb7f8bd880f005abc0f1287b1f9ebd48f63bcf914d3bf326a7dc5b125071738cecb3e8228f1965c361c813c3a6d7ac474cc34ae0324d0d72f1e
6
+ metadata.gz: d8640e69644843b589c390ccc368c85b24dc29e42c01dbd379e309409e0e187630230f7606852fb15f70a9ec58342a5f4154fd86c5625dfab47637d9b14f6c47
7
+ data.tar.gz: 71a4de8644e8e1ed5d0191fea52ce57384e823ef83b8136d13756d79fec5dc56cf9ac969552d895458aa1410c91724018cc91142f38e0aa62c57ae4034e09ab3
data/Rakefile CHANGED
@@ -8,7 +8,7 @@ RSpec::Core::RakeTask.new(:spec)
8
8
  # Pin for `rake compile` and the platform-gem builds. Keep in lockstep
9
9
  # with .github/workflows/build.yml (which calls `rake compile`) and the
10
10
  # CHANGELOG when libleptris releases.
11
- LIBLEPTRIS_VERSION = "1.9.375"
11
+ LIBLEPTRIS_VERSION = "1.9.376"
12
12
  # Vendored alongside libleptris for fn:normalize-unicode (TODO
13
13
  # .restructure/20): built per platform with a RELOCATABLE @rpath
14
14
  # install name, loaded by ffi.rb before libleptris so the
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Leptris
4
- VERSION = "1.9.375.0"
4
+ VERSION = "1.9.376.0"
5
5
  end
data/vendor-src/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  Sources vendored in this gem (recompile rights; the ruby
2
2
  variant builds them at install via extconf.rb):
3
3
 
4
- - libleptris v1.9.375
4
+ - libleptris v1.9.376
5
5
  - utf8proc v2.11.0
6
6
 
7
7
  Rebuild by hand:
@@ -1,5 +1,45 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [1.9.376] - 2026-10-20
4
+
5
+ ### Fixed
6
+
7
+ - **serialize: decoded entity-shaped text is escaped
8
+ unconditionally (leptris-ruby#515, PR #1793).** Parse expands
9
+ entities into text (named references that survive under
10
+ `KEEP_ENTITY_REFS` become `ENTITY_REF` nodes), but both text
11
+ emitters kept an entity lookahead that re-emitted decoded data
12
+ verbatim: `A` reparsed as `A`, and `<UK>` minted a
13
+ `<UK>` element that swallowed siblings. Every `&` in text is now
14
+ escaped; `ENTITY_REF` nodes still serialize as markup.
15
+ - **parse: strict loads validate the effective encoding
16
+ (leptris-ruby#514, PR #1793).** Non-RECOVER parses reject
17
+ invalid UTF-8 under XML 1.0 §4.3.3 (RFC 3629 strictness:
18
+ overlongs, surrogates, >U+10FFFF, truncation are fatal), for
19
+ documents that are UTF-8 by declaration or default. A declared
20
+ non-UTF-8 encoding is exempt (legacy encodings still transcode
21
+ through the iconv lane), and RECOVER keeps the #1660
22
+ best-effort-tree contract. The ASCII fast path scans
23
+ word-at-a-time (~14 GB/s, ~0.2% of a full parse).
24
+
25
+ ### Added
26
+
27
+ - **html: verdict parity 1446 to 1508 of 1555 (PR #1794).** Seven
28
+ emission slices, each grounded in html5lib's source and gated by
29
+ the #1771 corpus harness with zero want=clean regressions:
30
+ depth-0 stray raw-text end tags (`</script>`, `</style>`, …);
31
+ later `<html>` tokens (non-html-root); the end-tag-too-early
32
+ family (heading mismatch, implied-end crossings, and the same
33
+ rule on li/dd/dt/p implied closes); after-body/frameset phase
34
+ codes; InBody frameset, duplicate body/head, and AfterHead
35
+ out-of-my-head tags; select/table/foreign codes
36
+ (unexpected-input-in-select, hidden-input-in-table,
37
+ unexpected-html-element-in-foreign-content); and tokenizer
38
+ need-space-after-doctype. Tree parity stays 1555/1555; the
39
+ verdict floor rose from 1409 to 1508.
40
+
41
+
42
+
3
43
  ## [1.9.375] - 2026-10-20
4
44
 
5
45
  ### Added
@@ -3,7 +3,7 @@
3
3
  cmake_minimum_required(VERSION 3.20)
4
4
 
5
5
  project(leptris
6
- VERSION 1.9.375
6
+ VERSION 1.9.376
7
7
  DESCRIPTION "Fast XML parser and XPath evaluator in pure C"
8
8
  LANGUAGES C CXX
9
9
  )
@@ -838,6 +838,11 @@ LEPTRIS_API LeptrisStatus leptris_document_set_root_ex(
838
838
  /**
839
839
  * Parse XML string into document
840
840
  *
841
+ * Encoding: the input must be valid in its effective encoding. With
842
+ * no XML declaration (or a UTF-8/US-ASCII one) that is UTF-8 (XML
843
+ * 1.0 §4.3.3) and invalid byte sequences are a fatal load error. A
844
+ * declared legacy encoding is transcoded to UTF-8 before parsing.
845
+ *
841
846
  * @param xml XML string (must be valid UTF-8)
842
847
  * @param length Length of XML string in bytes
843
848
  * @param status Output status code (can be NULL)
@@ -979,6 +984,11 @@ LEPTRIS_API int leptris_rng_error_line(LeptrisRelaxNG rng, size_t i);
979
984
  LEPTRIS_API int leptris_rng_error_column(LeptrisRelaxNG rng, size_t i);
980
985
 
981
986
 
987
+ /* Parse with explicit flags. Strict (non-RECOVER) parses validate
988
+ * the input against its effective encoding (see leptris_parse_string);
989
+ * RECOVER keeps the best-effort #1660 contract and skips the fatal
990
+ * encoding check. A declared non-UTF-8 encoding is exempt from UTF-8
991
+ * validation and passes through raw in this lane. */
982
992
  LEPTRIS_API LeptrisDocument leptris_parse_string_flags(const char* xml,
983
993
  size_t length,
984
994
  LeptrisParseFlags flags,
@@ -43,6 +43,7 @@
43
43
  #include <string.h>
44
44
  #include <stdlib.h>
45
45
  #include <stdio.h>
46
+ #include <stdint.h>
46
47
 
47
48
  /* File-scope externs: engine config TLS defined in core.c (declared
48
49
  * in leptris_internal.h; this TU reads it without the umbrella) and
@@ -3415,6 +3416,123 @@ static struct leptris_document* dp_il_try(const char* xml, size_t len,
3415
3416
  return d;
3416
3417
  }
3417
3418
 
3419
+ /* ---- Load-time encoding validation (leptris-ruby#514) ----
3420
+ *
3421
+ * XML 1.0 §4.3.3: a document without a declaration is UTF-8; a
3422
+ * declared non-UTF-8 encoding is exempt (its bytes are legitimately
3423
+ * not UTF-8 — leptris_parse_string transcodes declared legacy
3424
+ * encodings through iconv before they reach this funnel). Structural
3425
+ * UTF-8 (RFC 3629) is strict: overlongs, surrogates, >U+10FFFF and
3426
+ * truncated sequences are fatal, matching libxml2's load-time
3427
+ * "Invalid bytes in character encoding". */
3428
+
3429
+ static int dp_ascii_case_eq(const char* a, const char* b, size_t n) {
3430
+ for (size_t i = 0; i < n; i++) {
3431
+ char x = a[i];
3432
+ char y = b[i];
3433
+ if (x >= 'A' && x <= 'Z') x += 32;
3434
+ if (y >= 'A' && y <= 'Z') y += 32;
3435
+ if (x != y) return 0;
3436
+ }
3437
+ return 1;
3438
+ }
3439
+
3440
+ /* 1 when the XML declaration names an encoding other than UTF-8 /
3441
+ * US-ASCII. No (or unreadable) declaration -> 0, the UTF-8 default. */
3442
+ static int dp_declares_non_utf8(const char* s, size_t len) {
3443
+ size_t i = 0;
3444
+ if (len >= 3 && (unsigned char)s[0] == 0xEF &&
3445
+ (unsigned char)s[1] == 0xBB && (unsigned char)s[2] == 0xBF) {
3446
+ i = 3;
3447
+ }
3448
+ if (len - i < 5 || memcmp(s + i, "<?xml", 5) != 0) return 0;
3449
+ if (len - i > 5) {
3450
+ char c = s[i + 5];
3451
+ if (c != ' ' && c != '\t' && c != '\r' && c != '\n' && c != '?')
3452
+ return 0;
3453
+ }
3454
+ size_t j = i + 5;
3455
+ size_t end = j;
3456
+ while (end + 1 < len && !(s[end] == '?' && s[end + 1] == '>')) end++;
3457
+ if (end + 1 >= len) return 0;
3458
+ while (j < end) {
3459
+ if (end - j >= 8 && memcmp(s + j, "encoding", 8) == 0 &&
3460
+ (j == i + 5 || s[j - 1] == ' ' || s[j - 1] == '\t' ||
3461
+ s[j - 1] == '\r' || s[j - 1] == '\n')) {
3462
+ size_t k = j + 8;
3463
+ while (k < end && (s[k] == ' ' || s[k] == '\t' ||
3464
+ s[k] == '\r' || s[k] == '\n')) k++;
3465
+ if (k < end && s[k] == '=') {
3466
+ k++;
3467
+ while (k < end && (s[k] == ' ' || s[k] == '\t' ||
3468
+ s[k] == '\r' || s[k] == '\n')) k++;
3469
+ if (k < end && (s[k] == '"' || s[k] == '\'')) {
3470
+ char q = s[k++];
3471
+ size_t v = k;
3472
+ while (k < end && s[k] != q) k++;
3473
+ size_t vl = k - v;
3474
+ if (vl == 5 && dp_ascii_case_eq(s + v, "UTF-8", 5)) return 0;
3475
+ if (vl == 4 && dp_ascii_case_eq(s + v, "UTF8", 4)) return 0;
3476
+ if (vl == 8 && dp_ascii_case_eq(s + v, "US-ASCII", 8))
3477
+ return 0;
3478
+ return 1;
3479
+ }
3480
+ }
3481
+ }
3482
+ j++;
3483
+ }
3484
+ return 0;
3485
+ }
3486
+
3487
+ static int dp_utf8_valid(const char* s, size_t len) {
3488
+ const unsigned char* p = (const unsigned char*)s;
3489
+ size_t i = 0;
3490
+ /* ASCII fast path: 8 bytes per iteration. The DFA below runs
3491
+ * only when a high byte is present; ordinary documents pay one
3492
+ * memory-bandwidth pass. */
3493
+ while (i + 8 <= len) {
3494
+ uint64_t w;
3495
+ memcpy(&w, p + i, 8);
3496
+ if (w & UINT64_C(0x8080808080808080)) break;
3497
+ i += 8;
3498
+ }
3499
+ while (i < len && p[i] < 0x80) i++;
3500
+ while (i < len) {
3501
+ unsigned c = p[i];
3502
+ if (c < 0x80) {
3503
+ i++;
3504
+ continue;
3505
+ }
3506
+ size_t n;
3507
+ unsigned lo = 0x80;
3508
+ unsigned hi = 0xBF;
3509
+ if (c >= 0xC2 && c <= 0xDF) {
3510
+ n = 1;
3511
+ } else if (c >= 0xE0 && c <= 0xEF) {
3512
+ n = 2;
3513
+ if (c == 0xE0) lo = 0xA0;
3514
+ else if (c == 0xED) hi = 0x9F;
3515
+ } else if (c >= 0xF0 && c <= 0xF4) {
3516
+ n = 3;
3517
+ if (c == 0xF0) lo = 0x90;
3518
+ else if (c == 0xF4) hi = 0x8F;
3519
+ } else {
3520
+ return 0;
3521
+ }
3522
+ if (len - i <= n) return 0;
3523
+ if (p[i + 1] < lo || p[i + 1] > hi) return 0;
3524
+ for (size_t k = 2; k <= n; k++) {
3525
+ if (p[i + k] < 0x80 || p[i + k] > 0xBF) return 0;
3526
+ }
3527
+ i += n + 1;
3528
+ }
3529
+ return 1;
3530
+ }
3531
+
3532
+ static int dp_input_encoding_ok(const char* s, size_t len) {
3533
+ return dp_declares_non_utf8(s, len) || dp_utf8_valid(s, len);
3534
+ }
3535
+
3418
3536
  /* Public: copy the input then parse (standard path).
3419
3537
  *
3420
3538
  * The copy and the arena-sizing pre-scan are ONE fused SIMD pass
@@ -3424,6 +3542,7 @@ static struct leptris_document* dp_il_try(const char* xml, size_t len,
3424
3542
  * arena holds only nodes and strings. */
3425
3543
  struct leptris_document* direct_parse(const char* xml, size_t len) {
3426
3544
  if (!xml || len == 0) return NULL;
3545
+ if (!dp_input_encoding_ok(xml, len)) return NULL;
3427
3546
  {
3428
3547
  struct leptris_document* d = dp_il_try(xml, len, 0, 0, 0);
3429
3548
  if (d) return d;
@@ -3435,6 +3554,12 @@ struct leptris_document* direct_parse(const char* xml, size_t len) {
3435
3554
  struct leptris_document* direct_parse_flags(const char* xml, size_t len,
3436
3555
  unsigned parse_flags) {
3437
3556
  if (!xml || len == 0) return NULL;
3557
+ /* RECOVER keeps the #1660 best-effort contract (never NULL for
3558
+ * malformed input); the fatal encoding check is strict-lane. */
3559
+ if (!(parse_flags & LEPTRIS_PARSE_RECOVER) &&
3560
+ !dp_input_encoding_ok(xml, len)) {
3561
+ return NULL;
3562
+ }
3438
3563
  /* The interleaved lane has no recovery support (#1660) — RECOVER
3439
3564
  * parses take the default lane. */
3440
3565
  if (!(parse_flags & LEPTRIS_PARSE_RECOVER)) {
@@ -3462,6 +3587,7 @@ struct leptris_document* direct_parse_flags(const char* xml, size_t len,
3462
3587
  /* Public: parse a caller-owned writable buffer without copying. */
3463
3588
  struct leptris_document* direct_parse_inplace(char* buf, size_t len) {
3464
3589
  if (!buf || len == 0) return NULL;
3590
+ if (!dp_input_encoding_ok(buf, len)) return NULL;
3465
3591
  buf[len] = '\0'; /* Ensure NUL termination */
3466
3592
  return direct_parse_internal(buf, len, 0, 0, 0, 0, 0, 0);
3467
3593
  }
@@ -2767,6 +2767,9 @@ typedef struct {
2767
2767
  * switch, so the divert stays armed. Re-armed by each
2768
2768
  * </body>/</html> token. */
2769
2769
  int after_body_done;
2770
+ /* Verdict bookkeeping only (never gates tree placement): the
2771
+ * AfterBody startTagOther error has fired for this phase. */
2772
+ int after_body_diag_done;
2770
2773
  /* #659 two-mode split: leptris_parse_html_string is the WHATWG
2771
2774
  * engine (full "in head" set: script/style/noscript/template
2772
2775
  * ... lift into the implied head); the new
@@ -3299,6 +3302,53 @@ static int h_is_formatting(const char* n) {
3299
3302
  return h_fmt4_lut[h_tag_id(n)];
3300
3303
  }
3301
3304
 
3305
+ /* Raw-text tokenizer targets. In body they have no dedicated end-tag
3306
+ * handler: html5lib's endTagOther walks the stack and reports
3307
+ * unexpected-end-tag at the first special node, so every such token
3308
+ * reaching the generic walk is stray (the tokenizer consumes the
3309
+ * matching terminator when the element is actually open). */
3310
+ static int h_is_rawtext_name(const char* n) {
3311
+ return strcmp(n, "script") == 0 || strcmp(n, "style") == 0 ||
3312
+ strcmp(n, "title") == 0 || strcmp(n, "textarea") == 0 ||
3313
+ strcmp(n, "xmp") == 0 || strcmp(n, "iframe") == 0 ||
3314
+ strcmp(n, "noembed") == 0 || strcmp(n, "noframes") == 0;
3315
+ }
3316
+
3317
+ /* html5lib generateImpliedEndTags set (§13.2.4.5): closing a block
3318
+ * or list item may pop these silently; anything else above the
3319
+ * target makes the close "too early". */
3320
+ static int h_is_implied_end_name(const char* n) {
3321
+ return strcmp(n, "p") == 0 || strcmp(n, "li") == 0 ||
3322
+ strcmp(n, "dd") == 0 || strcmp(n, "dt") == 0 ||
3323
+ strcmp(n, "option") == 0 || strcmp(n, "optgroup") == 0 ||
3324
+ strcmp(n, "rb") == 0 || strcmp(n, "rt") == 0 ||
3325
+ strcmp(n, "rtc") == 0 || strcmp(n, "rp") == 0;
3326
+ }
3327
+
3328
+ /* End tags html5lib implements with endTagBlock / endTagListItem /
3329
+ * endTagAppletMarqueeObject — the handlers that raise
3330
+ * end-tag-too-early when the target is not the current node after
3331
+ * implied-end generation. */
3332
+ static int h_is_tooearly_target(const char* n) {
3333
+ return strcmp(n, "p") == 0 || strcmp(n, "li") == 0 ||
3334
+ strcmp(n, "dd") == 0 || strcmp(n, "dt") == 0 ||
3335
+ strcmp(n, "applet") == 0 || strcmp(n, "marquee") == 0 ||
3336
+ strcmp(n, "object") == 0 ||
3337
+ strcmp(n, "address") == 0 || strcmp(n, "article") == 0 ||
3338
+ strcmp(n, "aside") == 0 || strcmp(n, "blockquote") == 0 ||
3339
+ strcmp(n, "button") == 0 || strcmp(n, "center") == 0 ||
3340
+ strcmp(n, "details") == 0 || strcmp(n, "dialog") == 0 ||
3341
+ strcmp(n, "dir") == 0 || strcmp(n, "div") == 0 ||
3342
+ strcmp(n, "dl") == 0 || strcmp(n, "fieldset") == 0 ||
3343
+ strcmp(n, "figcaption") == 0 || strcmp(n, "figure") == 0 ||
3344
+ strcmp(n, "footer") == 0 || strcmp(n, "header") == 0 ||
3345
+ strcmp(n, "hgroup") == 0 || strcmp(n, "listing") == 0 ||
3346
+ strcmp(n, "main") == 0 || strcmp(n, "menu") == 0 ||
3347
+ strcmp(n, "nav") == 0 || strcmp(n, "ol") == 0 ||
3348
+ strcmp(n, "pre") == 0 || strcmp(n, "section") == 0 ||
3349
+ strcmp(n, "summary") == 0 || strcmp(n, "ul") == 0;
3350
+ }
3351
+
3302
3352
  /* WHATWG "special" category (13.2.4.2) — the adoption agency's
3303
3353
  * furthest-block candidates. */
3304
3354
  /* Generated tag classifier (#1218 hot path): one first-char
@@ -4133,6 +4183,17 @@ static void h_append(HBuilder* b, LeptrisNodeRef n) {
4133
4183
  b->open_ns[b->depth - 2] == H_NS_HTML &&
4134
4184
  b->open_id[b->depth - 2] == h_id_table)
4135
4185
  tbl_ctx = 1;
4186
+ /* InTable startTagInput with type=hidden: html5lib reports
4187
+ * unexpected-hidden-input-in-table and inserts AT the spot
4188
+ * (no fostering — h_fosterable exempts it, tests7:16-20). */
4189
+ if (b->whatwg_foster && tbl_ctx && b->input_hidden_tag &&
4190
+ leptris_node_get_type(n) == LEPTRIS_NODE_TYPE_ELEMENT &&
4191
+ h_tag_id(leptris_element_name((LeptrisElement)n)) ==
4192
+ h_id_input) {
4193
+ h_diag(b, "unexpected-hidden-input-in-table",
4194
+ b->err_line_memo_off,
4195
+ "hidden input inserted in table context");
4196
+ }
4136
4197
  /* #659 foster (WHATWG only): text/elements in table context
4137
4198
  * go before the nearest open table in ITS parent. */
4138
4199
  if (b->whatwg_foster && tbl_ctx && h_fosterable(b, n)) {
@@ -5851,6 +5912,13 @@ static LeptrisDocument html_parse_shared(
5851
5912
  }
5852
5913
  if (is_dt) {
5853
5914
  q += 7;
5915
+ /* Tokenizer: need-space-after-doctype when the
5916
+ * keyword is not followed by whitespace
5917
+ * (doctype01:3, tests2:26-28). */
5918
+ if (b.whatwg && q < end && *q != '>' && !h_is_ws(*q))
5919
+ h_diag(&b, "need-space-after-doctype",
5920
+ (size_t)(q - b.input_base),
5921
+ "no space after <!doctype");
5854
5922
  while (q < end && h_is_ws(*q)) q++;
5855
5923
  const char* nstart = q;
5856
5924
  while (q < end && !h_is_ws(*q) && *q != '>') q++;
@@ -6191,6 +6259,27 @@ static LeptrisDocument html_parse_shared(
6191
6259
  text = p;
6192
6260
  continue;
6193
6261
  }
6262
+ /* Stray raw-text end tag with no open stack: the raw-text
6263
+ * element completed during its start-tag handling, so this
6264
+ * token cannot close anything. html5lib's InHead/InBody
6265
+ * endTagOther reports unexpected-end-tag once the tree has
6266
+ * content (the synthesized body is the special node);
6267
+ * with nothing seen yet the phase errors differently and
6268
+ * the initial-mode checks above already cover it. */
6269
+ if (b.whatwg && nlen && b.depth == 0 && b.top_tail) {
6270
+ char rn[16];
6271
+ size_t rl = nlen < sizeof(rn) - 1 ? nlen : sizeof(rn) - 1;
6272
+ for (size_t i = 0; i < rl; i++) rn[i] = h_lower(ns[i]);
6273
+ rn[rl] = 0;
6274
+ if (h_is_rawtext_name(rn)) {
6275
+ h_diag(&b, "unexpected-end-tag",
6276
+ (size_t)(p - b.input_base),
6277
+ "</%s> has no matching open element", rn);
6278
+ p = q;
6279
+ text = p;
6280
+ continue;
6281
+ }
6282
+ }
6194
6283
  if (nlen && (b.depth > 0 ||
6195
6284
  (b.whatwg &&
6196
6285
  ((nlen == 4 &&
@@ -6274,6 +6363,29 @@ static LeptrisDocument html_parse_shared(
6274
6363
  const char* hn =
6275
6364
  leptris_element_name(b.open[d - 1]);
6276
6365
  if (hn && h_is_heading(hn)) {
6366
+ /* html5lib endTagHeading: after the
6367
+ * implied-end pass, a current
6368
+ * heading of a different name makes
6369
+ * the close too early (tests19:24:
6370
+ * </h2> over an open h3), and so
6371
+ * does a non-implied element above
6372
+ * the heading. */
6373
+ int early = strcmp(hn, lname) != 0;
6374
+ for (size_t k = d; k < b.depth; k++) {
6375
+ const char* an2 = leptris_element_name(
6376
+ b.open[k]);
6377
+ if (an2 &&
6378
+ b.open_ns[k] == H_NS_HTML &&
6379
+ !h_is_implied_end_name(an2)) {
6380
+ early = 1;
6381
+ break;
6382
+ }
6383
+ }
6384
+ if (early)
6385
+ h_diag(&b, "end-tag-too-early",
6386
+ (size_t)(p - b.input_base),
6387
+ "</%s> closes <%s> early",
6388
+ lname, hn);
6277
6389
  h_pop_to(&b, d - 1);
6278
6390
  /* Old-suite rule (tests1:21/91):
6279
6391
  * </table> also closes open
@@ -6314,6 +6426,12 @@ static LeptrisDocument html_parse_shared(
6314
6426
  text = p;
6315
6427
  continue;
6316
6428
  }
6429
+ /* No heading in scope: html5lib still checks
6430
+ * the current node, which is never the token
6431
+ * (tests19:23: </h3> after <p>). */
6432
+ h_diag(&b, "end-tag-too-early",
6433
+ (size_t)(p - b.input_base),
6434
+ "</%s> with no open heading", lname);
6317
6435
  }
6318
6436
  /* #659 13.2.6.4.17: </p> with no p in button
6319
6437
  * scope inserts an EMPTY <p> and closes it
@@ -6623,7 +6741,14 @@ static LeptrisDocument html_parse_shared(
6623
6741
  strcmp(lname, "dd") == 0 ||
6624
6742
  strcmp(lname, "dt") == 0) &&
6625
6743
  (h_ieq_raw(on, "ul") ||
6626
- h_ieq_raw(on, "ol")));
6744
+ h_ieq_raw(on, "ol"))) ||
6745
+ /* html5lib endTagOther: a
6746
+ * raw-text name has no
6747
+ * in-body handler, so the
6748
+ * walk errors at the first
6749
+ * special node. */
6750
+ (h_is_rawtext_name(lname) &&
6751
+ h_is_special_ww(on));
6627
6752
  }
6628
6753
  if (fenced) {
6629
6754
  /* #1771: html5lib's walk reports
@@ -6677,6 +6802,27 @@ static LeptrisDocument html_parse_shared(
6677
6802
  (int)(d - 1) <= ti)
6678
6803
  break;
6679
6804
  }
6805
+ /* html5lib endTagBlock / endTagListItem /
6806
+ * endTagAppletMarqueeObject: after the
6807
+ * implied-end pass, a non-implied element
6808
+ * above the target makes the close too
6809
+ * early (tests3:11: </pre> over <div>). */
6810
+ if (b.whatwg && b.open_ns[d - 1] == H_NS_HTML &&
6811
+ h_is_tooearly_target(lname)) {
6812
+ for (size_t k = d; k < b.depth; k++) {
6813
+ const char* an2 =
6814
+ leptris_element_name(b.open[k]);
6815
+ if (an2 &&
6816
+ b.open_ns[k] == H_NS_HTML &&
6817
+ !h_is_implied_end_name(an2)) {
6818
+ h_diag(&b, "end-tag-too-early",
6819
+ (size_t)(p - b.input_base),
6820
+ "</%s> closes over <%s>",
6821
+ lname, an2);
6822
+ break;
6823
+ }
6824
+ }
6825
+ }
6680
6826
  h_pop_to(&b, d - 1);
6681
6827
  /* Old-suite </table> rule: closes
6682
6828
  * formatting BELOW the table - only
@@ -6977,6 +7123,47 @@ static LeptrisDocument html_parse_shared(
6977
7123
  while (q < end && !h_is_ws(*q) && *q != '>' && *q != '/') q++;
6978
7124
  size_t nlen = (size_t)(q - ns);
6979
7125
  char* name = h_pooled_lower(b.pool, ns, nlen);
7126
+ if (name && b.whatwg && strcmp(name, "html") != 0) {
7127
+ if (b.after_html &&
7128
+ !(b.frameset && strcmp(name, "noframes") == 0)) {
7129
+ /* AfterAfterBody / AfterAfterFrameset startTagOther
7130
+ * (only html — and noframes in the frameset flavor —
7131
+ * is accepted). */
7132
+ h_diag(&b, "expected-eof-but-got-start-tag",
7133
+ (size_t)(p - b.input_base),
7134
+ "<%s> after </html>", name);
7135
+ } else if (b.after_body && !b.after_body_done &&
7136
+ !b.frameset && !b.after_body_diag_done) {
7137
+ /* AfterBody startTagOther: the first non-html start
7138
+ * tag after </body> errors and moves the phase back
7139
+ * to in-body (tests15:4/6, tests9:21, tests10:20).
7140
+ * Bookkeeping is diagnostic-only: after_body_done
7141
+ * also gates comment placement. */
7142
+ h_diag(&b, "unexpected-start-tag-after-body",
7143
+ (size_t)(p - b.input_base),
7144
+ "<%s> after </body>", name);
7145
+ b.after_body_diag_done = 1;
7146
+ }
7147
+ /* AfterHead startTagFromHead: a head-only start tag in
7148
+ * the window after </head> and before body content is
7149
+ * processed in head but reported out-of-my-head
7150
+ * (tests7:3/4). */
7151
+ if (b.head_end_seen && !b.body_seen && !b.body_tag_seen &&
7152
+ !b.frameset && !b.after_body &&
7153
+ (strcmp(name, "base") == 0 ||
7154
+ strcmp(name, "basefont") == 0 ||
7155
+ strcmp(name, "bgsound") == 0 ||
7156
+ strcmp(name, "link") == 0 ||
7157
+ strcmp(name, "meta") == 0 ||
7158
+ strcmp(name, "noframes") == 0 ||
7159
+ strcmp(name, "script") == 0 ||
7160
+ strcmp(name, "style") == 0 ||
7161
+ strcmp(name, "title") == 0)) {
7162
+ h_diag(&b, "unexpected-start-tag-out-of-my-head",
7163
+ (size_t)(p - b.input_base),
7164
+ "<%s> after </head>", name);
7165
+ }
7166
+ }
6980
7167
  if (!name) goto done;
6981
7168
  /* #659 (13.2.6.4.7): <image> is renamed <img> and
6982
7169
  * reprocessed (tests1:90). */
@@ -7070,6 +7257,13 @@ static LeptrisDocument html_parse_shared(
7070
7257
  if (t) h_append(&b, (LeptrisNodeRef)t);
7071
7258
  }
7072
7259
  if (strcmp(name, "body") == 0) {
7260
+ /* InBody startTagBody always reports
7261
+ * unexpected-start-tag for a second <body>
7262
+ * (tests2:16); the first one is the phase entry. */
7263
+ if (b.whatwg && (b.body_tag_seen || b.body_seen))
7264
+ h_diag(&b, "unexpected-start-tag",
7265
+ (size_t)(p - b.input_base),
7266
+ "<body> after the body exists");
7073
7267
  b.body_tag_seen = 1;
7074
7268
  if (b.whatwg) b.frameset_ok = 0;
7075
7269
  /* Only the FIRST structural <body> marks the
@@ -7081,6 +7275,15 @@ static LeptrisDocument html_parse_shared(
7081
7275
  b.lift_boundary = b.top_tail;
7082
7276
  }
7083
7277
  } else {
7278
+ /* InBody/AfterHead startTagHead: any <head> after
7279
+ * the head phase started is an unexpected start tag
7280
+ * (tests6:1: </head> <head>). */
7281
+ if (b.whatwg &&
7282
+ (b.head_tag_seen || b.head_end_seen ||
7283
+ b.body_tag_seen || b.body_seen))
7284
+ h_diag(&b, "unexpected-start-tag",
7285
+ (size_t)(p - b.input_base),
7286
+ "<head> after the head phase");
7084
7287
  /* Before-head ends here: later comments are head
7085
7288
  * content; earlier ones stay html-prefix children. */
7086
7289
  b.head_tag_seen = 1;
@@ -7109,18 +7312,18 @@ static LeptrisDocument html_parse_shared(
7109
7312
  /* frameset is NOT in the 13.2.6.5 breakout list: inside
7110
7313
  * foreign content it is a plain foreign element
7111
7314
  * (plain-text-unsafe:16-23). */
7112
- /* #1771: html5lib's verdict — a <frameset> token that
7113
- * reaches the body phase after body content is
7114
- * unexpected-start-tag whether it then converts or is
7115
- * ignored (webkit01:51); head-phase converters and
7116
- * in-template framesets stay clean. body_seen is not
7117
- * set by the frameset token itself (it is in the
7118
- * head-only list), so it reads PRIOR content only. */
7119
- if (b.body_seen && !b.frameset &&
7315
+ /* #1771: html5lib's InBody startTagFrameset always
7316
+ * reports unexpected-start-tag — the frameset either
7317
+ * converts (body element absent, frameset-ok set) or is
7318
+ * ignored. frameset_ok==0 marks the InBody phase: any
7319
+ * body tag/content (or a frameset-ok-clearing start)
7320
+ * has already been seen. AfterHead converters keep
7321
+ * frameset_ok==1 and stay clean (webkit01:51). */
7322
+ if (!b.frameset_ok && !b.frameset &&
7120
7323
  h_template_idx(&b) < 0)
7121
7324
  h_diag(&b, "unexpected-start-tag",
7122
7325
  (size_t)(p - b.input_base),
7123
- "frameset after body content");
7326
+ "frameset in the body phase");
7124
7327
  /* depth<=1: at most the (explicit or synthesized) html
7125
7328
  * element is open; frameset-ok is the spec gate and the
7126
7329
  * body is removed wholesale when it converts. */
@@ -7195,6 +7398,20 @@ static LeptrisDocument html_parse_shared(
7195
7398
  strcmp(name, "frame") != 0 &&
7196
7399
  strcmp(name, "noframes") != 0 &&
7197
7400
  strcmp(name, "html") != 0) {
7401
+ /* html5lib phase verdicts for the dropped token (the
7402
+ * after-</html> flavor is emitted at the start-tag
7403
+ * dispatch): AfterFrameset names its own code; inside an
7404
+ * open frameset it is InFrameset's. */
7405
+ if (!b.after_html) {
7406
+ if (b.after_frameset)
7407
+ h_diag(&b, "unexpected-start-tag-after-frameset",
7408
+ (size_t)(p - b.input_base),
7409
+ "<%s> after </frameset>", name);
7410
+ else
7411
+ h_diag(&b, "unexpected-start-tag-in-frameset",
7412
+ (size_t)(p - b.input_base),
7413
+ "<%s> inside <frameset>", name);
7414
+ }
7198
7415
  if (text < p) {
7199
7416
  LeptrisTextNode* t = h_text_node(&b, text, p);
7200
7417
  if (t) {
@@ -7329,6 +7546,12 @@ static LeptrisDocument html_parse_shared(
7329
7546
  (strcmp(name, "input") == 0 ||
7330
7547
  strcmp(name, "keygen") == 0 ||
7331
7548
  strcmp(name, "textarea") == 0)) {
7549
+ /* InSelect startTagInput (shared by input/keygen/
7550
+ * textarea) reports before popping the select
7551
+ * (tests7:14). */
7552
+ h_diag(&b, "unexpected-input-in-select",
7553
+ (size_t)(p - b.input_base),
7554
+ "<%s> inside <select>", name);
7332
7555
  for (size_t d2 = b.depth; d2 > 0; d2--)
7333
7556
  if (strcmp(leptris_element_name(b.open[d2 - 1]),
7334
7557
  "select") == 0) {
@@ -7383,6 +7606,17 @@ static LeptrisDocument html_parse_shared(
7383
7606
  continue;
7384
7607
  }
7385
7608
 
7609
+ /* html5lib startTagHtml: an <html> token processed once the
7610
+ * html/head/body phases exist is not the first start tag —
7611
+ * parse error non-html-root, then the attributes merge onto
7612
+ * the root (tests14:3/4/6, tests19:37/38/88, tests2:52-54). */
7613
+ if (b.whatwg && strcmp(name, "html") == 0 &&
7614
+ (b.html_seen || b.head_tag_seen || b.body_tag_seen ||
7615
+ b.after_body || b.body_seen)) {
7616
+ h_diag(&b, "non-html-root",
7617
+ (size_t)(p - b.input_base),
7618
+ "<html> after the root element exists");
7619
+ }
7386
7620
  /* 13.2.6.3: an <html> start tag inside content (never
7387
7621
  * seen yet, deep in a table/foreign subtree) merges attrs
7388
7622
  * and is ignored - no element (domjs-unsafe:36:
@@ -7502,7 +7736,13 @@ static LeptrisDocument html_parse_shared(
7502
7736
  if (h_is_breakout(name) ||
7503
7737
  (strcmp(name, "font") == 0 && h_font_break(q, end))) {
7504
7738
  /* Breakout: pop the foreign scope, reprocess
7505
- * under HTML rules below. */
7739
+ * under HTML rules below. html5lib's foreign
7740
+ * startTagOther reports the HTML element as
7741
+ * unexpected in foreign content
7742
+ * (tests9:15, tests10:14). */
7743
+ h_diag(&b, "unexpected-html-element-in-foreign-content",
7744
+ (size_t)(p - b.input_base),
7745
+ "<%s> breaks out of foreign content", name);
7506
7746
  while (b.depth > 0 &&
7507
7747
  b.open_ns[b.depth - 1] != H_NS_HTML &&
7508
7748
  !h_is_int_point(&b, b.depth - 1))
@@ -7520,6 +7760,12 @@ static LeptrisDocument html_parse_shared(
7520
7760
  /* Implied end tags this start tag triggers (HTML rules
7521
7761
  * only — foreign content has none). */
7522
7762
  if (elem_ns == H_NS_HTML) {
7763
+ /* html5lib startTagListItem/startTagBlock: an implied
7764
+ * close whose target is buried under a non-implied
7765
+ * element (div, article, ...) reports end-tag-too-early
7766
+ * on this START tag (tests2:11, tests3:20). One per
7767
+ * token, like the reference walk. */
7768
+ const char* implied_early = NULL;
7523
7769
  /* WHATWG: a dd/dt start closes the NEAREST open dd/dt
7524
7770
  * in default scope — sibling definition items even
7525
7771
  * with content in between (13.2.6.4.7, tests19:30:
@@ -7529,6 +7775,15 @@ static LeptrisDocument html_parse_shared(
7529
7775
  for (size_t d = b.depth; d > 0; d--) {
7530
7776
  uint8_t oid = b.open_id[d - 1];
7531
7777
  if (oid == h_id_dd || oid == h_id_dt) {
7778
+ for (size_t k = d; k < b.depth; k++) {
7779
+ const char* an2 =
7780
+ leptris_element_name(b.open[k]);
7781
+ if (an2 && b.open_ns[k] == H_NS_HTML &&
7782
+ !h_is_implied_end_name(an2)) {
7783
+ implied_early = an2;
7784
+ break;
7785
+ }
7786
+ }
7532
7787
  b.depth = d - 1;
7533
7788
  break;
7534
7789
  }
@@ -7550,6 +7805,17 @@ static LeptrisDocument html_parse_shared(
7550
7805
  if ((tid == h_id_li && oid == h_id_li) ||
7551
7806
  (tid != h_id_li &&
7552
7807
  (oid == h_id_dd || oid == h_id_dt))) {
7808
+ if (!implied_early) {
7809
+ for (size_t k = d; k < b.depth; k++) {
7810
+ const char* an2 =
7811
+ leptris_element_name(b.open[k]);
7812
+ if (an2 && b.open_ns[k] == H_NS_HTML &&
7813
+ !h_is_implied_end_name(an2)) {
7814
+ implied_early = an2;
7815
+ break;
7816
+ }
7817
+ }
7818
+ }
7553
7819
  b.depth = d - 1;
7554
7820
  break;
7555
7821
  }
@@ -7570,6 +7836,17 @@ static LeptrisDocument html_parse_shared(
7570
7836
  for (size_t d = b.depth; d > 0; d--) {
7571
7837
  uint8_t oid = b.open_id[d - 1];
7572
7838
  if (oid == h_id_p) {
7839
+ if (!implied_early) {
7840
+ for (size_t k = d; k < b.depth; k++) {
7841
+ const char* an2 =
7842
+ leptris_element_name(b.open[k]);
7843
+ if (an2 && b.open_ns[k] == H_NS_HTML &&
7844
+ !h_is_implied_end_name(an2)) {
7845
+ implied_early = an2;
7846
+ break;
7847
+ }
7848
+ }
7849
+ }
7573
7850
  b.depth = d - 1;
7574
7851
  break;
7575
7852
  }
@@ -7578,6 +7855,10 @@ static LeptrisDocument html_parse_shared(
7578
7855
  break;
7579
7856
  }
7580
7857
  }
7858
+ if (b.whatwg && implied_early)
7859
+ h_diag(&b, "end-tag-too-early",
7860
+ (size_t)(p - b.input_base),
7861
+ "implied close crosses <%s>", implied_early);
7581
7862
  uint8_t tmpl_last_popped = H_ID_UNKNOWN;
7582
7863
  while (b.depth > 0) {
7583
7864
  uint8_t oid = b.open_id[b.depth - 1];
@@ -203,10 +203,12 @@ int buffer_has_error(SerializeBuffer* buf) {
203
203
 
204
204
 
205
205
  /* Inline escaped-text emitter (TODO 194d): writes into a caller-
206
- * reserved worst-case buffer (6 bytes per input byte), preserving
207
- * the entity semantics of the previous per-run walkers — entity
208
- * references emit as-is (text mode), bare & escapes, quotes are
209
- * ordinary in text mode and escaped in attribute mode. */
206
+ * reserved worst-case buffer (6 bytes per input byte). Every '&' is
207
+ * escaped — entity references are parse-time markup and either
208
+ * already decoded into the text (predefined/char refs) or carried as
209
+ * ENTITY_REF nodes; a decoded text value shaped like "&#x26;" is
210
+ * data, not markup (leptris-ruby#515). Quotes are ordinary in text
211
+ * mode and escaped in attribute mode. */
210
212
  /* Escape-class table (serialize endgame): the run scan was 3-4
211
213
  * compares per byte — the dominant serializer cost at 57% of the
212
214
  * text-heavy profile. One indexed load + one test per byte, the
@@ -300,23 +302,9 @@ static char* emit_escaped_inline(char* out, const char* content,
300
302
  }
301
303
 
302
304
  if (content[i] == '&') {
303
- size_t j = i + 1;
304
- int found_semicolon = 0;
305
- while (j < len && j < i + 12) {
306
- if (content[j] == ';') { found_semicolon = 1; break; }
307
- if (!isalnum((unsigned char)content[j]) && content[j] != '#' && content[j] != '-') break;
308
- j++;
309
- }
310
- if (!attr_mode && found_semicolon && j > i + 1) {
311
- size_t n = j - i + 1;
312
- memcpy(out, &content[i], n);
313
- out += n;
314
- i = j + 1;
315
- } else {
316
- memcpy(out, "&amp;", 5);
317
- out += 5;
318
- i++;
319
- }
305
+ memcpy(out, "&amp;", 5);
306
+ out += 5;
307
+ i++;
320
308
  continue;
321
309
  }
322
310
  if (content[i] == '<') { memcpy(out, "&lt;", 4); out += 4; }
@@ -716,8 +704,8 @@ void serialize_text_internal(LeptrisTextNode* text, SerializeBuffer* buf) {
716
704
  }
717
705
 
718
706
  /* Run-batched (TODO 194b): bulk-append ordinary bytes up to the
719
- * next special character; the entity lookahead only runs at '&'.
720
- * Quotes and apostrophes are ordinary in text content. */
707
+ * next special character. Quotes and apostrophes are ordinary in
708
+ * text content. */
721
709
  size_t i = 0;
722
710
  while (i < content_len) {
723
711
  size_t run = i;
@@ -733,26 +721,10 @@ void serialize_text_internal(LeptrisTextNode* text, SerializeBuffer* buf) {
733
721
  }
734
722
 
735
723
  if (content[i] == '&') {
736
- /* Look ahead for ';' to detect entity reference */
737
- size_t j = i + 1;
738
- int found_semicolon = 0;
739
- while (j < text->content_len && j < i + 12) {
740
- if (content[j] == ';') {
741
- found_semicolon = 1;
742
- break;
743
- }
744
- if (!isalnum((unsigned char)content[j]) && content[j] != '#' && content[j] != '-') {
745
- break;
746
- }
747
- j++;
748
- }
749
- if (found_semicolon && j > i + 1) {
750
- buffer_append_len(buf, &content[i], j - i + 1);
751
- i = j + 1;
752
- } else {
753
- buffer_append_len(buf, "&amp;", 5);
754
- i++;
755
- }
724
+ /* Always escape: decoded text is data. Named references
725
+ * that survived parsing live in ENTITY_REF nodes. */
726
+ buffer_append_len(buf, "&amp;", 5);
727
+ i++;
756
728
  continue;
757
729
  }
758
730
 
@@ -977,9 +977,19 @@ TEST(Html5LibCorpus, ErrorVerdictParity) {
977
977
  for (size_t i = 0; i < failures.size() && i < 20; i++)
978
978
  printf(" VERDICT %s\n", failures[i].c_str());
979
979
  EXPECT_GT(total, (size_t)1500);
980
- /* Verdict-parity floor — set at the slice-2 baseline
981
- * (1409/1555; was 190 before the #1771 emission slices —
982
- * the doctype-less initial-mode family, trailing chars, EOF with
983
- * open elements; slice 3 adds the raw-text EOF variants). Only raises. */
984
- EXPECT_GE(parity, (size_t)1409);
980
+ /* Verdict-parity floor — main measured 1446/1555 before these
981
+ * slices (1409 was the slice-2 baseline; was 190 before the
982
+ * #1771 emission slices). Raw-text stray end tags add 12 depth-0
983
+ * cases; non-html-root adds the 9 later-<html> cases; the
984
+ * end-tag-too-early slices add heading name mismatches, block
985
+ * closes over non-implied elements, and the same rule on the
986
+ * implied close of li/dd/dt/p start tags; the after-body and
987
+ * frameset phase slices add the AfterBody/AfterFrameset/
988
+ * AfterAfter* start-tag codes; the start-tag phase slice adds
989
+ * InBody frameset, duplicate body/head, and AfterHead
990
+ * out-of-my-head tags; the select/table/foreign slice adds
991
+ * unexpected-input-in-select, hidden-input-in-table, and
992
+ * unexpected-html-element-in-foreign-content; the tokenizer
993
+ * slice adds need-space-after-doctype. Only raises. */
994
+ EXPECT_GE(parity, (size_t)1508);
985
995
  }
@@ -257,6 +257,93 @@ TEST(EncodingEdgeCases, EmptyXmlElementRoundTrips) {
257
257
  leptris_document_free(doc);
258
258
  }
259
259
 
260
+ /* leptris-ruby#514: strict load validates the input against the
261
+ * effective encoding. Without a declaration (or with a UTF-8 one)
262
+ * the entity is UTF-8 in XML 1.0; invalid byte sequences are a
263
+ * fatal load error, matching libxml2's "Invalid bytes in character
264
+ * encoding". A declared non-UTF-8 encoding is exempt — those bytes
265
+ * are legitimately not UTF-8. */
266
+ TEST(EncodingUtf8Strict, RejectsInvalidBytesWithoutDeclaration) {
267
+ const char bad[] = "<r>\xC3\x28</r>";
268
+ LeptrisStatus st = LEPTRIS_OK;
269
+ LeptrisDocument doc = leptris_parse_string(bad, sizeof(bad) - 1, &st);
270
+ EXPECT_EQ(doc, nullptr);
271
+ EXPECT_EQ(st, LEPTRIS_ERROR_PARSE);
272
+ }
273
+
274
+ TEST(EncodingUtf8Strict, RejectsInvalidBytesWithUtf8Declaration) {
275
+ const char bad[] =
276
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?><r>\xC3\x28</r>";
277
+ LeptrisStatus st = LEPTRIS_OK;
278
+ LeptrisDocument doc = leptris_parse_string(bad, sizeof(bad) - 1, &st);
279
+ EXPECT_EQ(doc, nullptr);
280
+ EXPECT_EQ(st, LEPTRIS_ERROR_PARSE);
281
+
282
+ const char bad_lower[] =
283
+ "<?xml version='1.0' encoding='utf-8'?><r>\xC3\x28</r>";
284
+ LeptrisDocument doc2 = leptris_parse_string(
285
+ bad_lower, sizeof(bad_lower) - 1, &st);
286
+ EXPECT_EQ(doc2, nullptr);
287
+ }
288
+
289
+ TEST(EncodingUtf8Strict, RejectsEveryInvalidSequenceClass) {
290
+ struct { const char* xml; size_t len; } cases[] = {
291
+ {"<r>\xC0\x80</r>", 0}, /* overlong NUL */
292
+ {"<r>\xED\xA0\x80</r>", 0}, /* surrogate U+D800 */
293
+ {"<r>\xF5\x80\x80\x80</r>", 0}, /* > U+10FFFF */
294
+ {"<r>\x80</r>", 0}, /* lone continuation */
295
+ {"<r>abc\xC3", 0}, /* truncated at EOF */
296
+ {"<r a='\xC3\x28'/>", 0}, /* invalid in attribute */
297
+ };
298
+ for (auto& c : cases) {
299
+ size_t len = c.len ? c.len : std::strlen(c.xml);
300
+ LeptrisStatus st = LEPTRIS_OK;
301
+ LeptrisDocument doc = leptris_parse_string(c.xml, len, &st);
302
+ EXPECT_EQ(doc, nullptr) << "accepted invalid bytes";
303
+ if (doc) leptris_document_free(doc);
304
+ }
305
+ }
306
+
307
+ TEST(EncodingUtf8Strict, AcceptsValidMultibyte) {
308
+ const char xml[] = "<r>caf\xC3\xA9 \xE4\xB8\xAD \xF0\x9F\x98\x80</r>";
309
+ LeptrisStatus st = LEPTRIS_OK;
310
+ LeptrisDocument doc = leptris_parse_string(xml, std::strlen(xml), &st);
311
+ ASSERT_NE(doc, nullptr);
312
+ EXPECT_STREQ(leptris_element_text(leptris_document_root(doc)),
313
+ "caf\xC3\xA9 \xE4\xB8\xAD \xF0\x9F\x98\x80");
314
+ leptris_document_free(doc);
315
+ }
316
+
317
+ TEST(EncodingUtf8Strict, DeclaredNonUtf8IsExempt) {
318
+ /* ISO-8859-1 0xE9 is not valid UTF-8 but is valid in the
319
+ * declared encoding: the load must not reject it. Depending on
320
+ * the iconv build it either transcodes to UTF-8 ("café") or
321
+ * passes the raw declared byte through — both are documented
322
+ * shapes (#514); only the bogus UTF-8 rejection would fail. */
323
+ const char xml[] =
324
+ "<?xml version='1.0' encoding='ISO-8859-1'?><r>caf\xE9</r>";
325
+ LeptrisStatus st = LEPTRIS_OK;
326
+ LeptrisDocument doc = leptris_parse_string(xml, sizeof(xml) - 1, &st);
327
+ ASSERT_NE(doc, nullptr);
328
+ const char* text = leptris_element_text(leptris_document_root(doc));
329
+ ASSERT_NE(text, nullptr);
330
+ EXPECT_TRUE(std::string(text) == "caf\xC3\xA9" ||
331
+ std::string(text) == "caf\xE9")
332
+ << "unexpected content byte(s)";
333
+ leptris_document_free(doc);
334
+ }
335
+
336
+ TEST(EncodingUtf8Strict, RecoverLaneKeepsBestEffortTree) {
337
+ /* #1660: RECOVER never returns NULL for malformed input; the
338
+ * encoding check must not break that contract. */
339
+ const char bad[] = "<r>\xC3\x28</r>";
340
+ LeptrisStatus st = LEPTRIS_OK;
341
+ LeptrisDocument doc = leptris_parse_string_flags(
342
+ bad, sizeof(bad) - 1, LEPTRIS_PARSE_RECOVER, &st);
343
+ ASSERT_NE(doc, nullptr);
344
+ leptris_document_free(doc);
345
+ }
346
+
260
347
  } // namespace
261
348
 
262
349
  /* Issue #613: with_encoding on declared ISO-8859-1 must deliver
@@ -39,6 +39,71 @@ TEST(SerializeRoundTrip, EscapesBareAmpersandInText) {
39
39
  leptris_document_free(doc);
40
40
  }
41
41
 
42
+ TEST(SerializeRoundTrip, EscapesEntityShapedTextUnconditionally) {
43
+ // leptris-ruby#515: parse decodes entities into text; a decoded
44
+ // text value that LOOKS like an entity reference ("&#x26;",
45
+ // "&amp;", "&lt;UK&gt;") must still escape its '&' on output.
46
+ // The old serializer passed entity-shaped runs through verbatim,
47
+ // so the next parse produced a different value (&#x41; -> A,
48
+ // &lt;UK&gt; -> a literal <UK> start tag).
49
+ struct { const char* xml; const char* want; const char* want_text; } cases[] = {
50
+ {"<r>a &amp;#x26; b</r>", "<r>a &amp;#x26; b</r>", "a &#x26; b"},
51
+ {"<r>a &amp;amp; b</r>", "<r>a &amp;amp; b</r>", "a &amp; b"},
52
+ {"<r>a &amp;lt;UK&amp;gt; b</r>", "<r>a &amp;lt;UK&amp;gt; b</r>",
53
+ "a &lt;UK&gt; b"},
54
+ {"<r>a &amp;#x41; b</r>", "<r>a &amp;#x41; b</r>", "a &#x41; b"},
55
+ // Attribute forces the non-fused text emitter; the first four
56
+ // cases take the fused <name>text</name> fast path.
57
+ {"<r k='v'>a &amp;#x26; b</r>", "<r k=\"v\">a &amp;#x26; b</r>",
58
+ "a &#x26; b"},
59
+ };
60
+ for (auto& c : cases) {
61
+ LeptrisStatus st = LEPTRIS_OK;
62
+ LeptrisDocument doc = leptris_parse_string(c.xml, std::strlen(c.xml), &st);
63
+ ASSERT_NE(doc, nullptr) << c.xml;
64
+ char* serialized = leptris_document_serialize(doc, nullptr);
65
+ ASSERT_NE(serialized, nullptr) << c.xml;
66
+ EXPECT_STREQ(serialized, c.want) << c.xml;
67
+
68
+ // The emitted bytes must reparse to the same string value.
69
+ LeptrisDocument again = leptris_parse_string(
70
+ serialized, std::strlen(serialized), &st);
71
+ ASSERT_NE(again, nullptr) << serialized;
72
+ LeptrisElement root = leptris_document_root(again);
73
+ ASSERT_NE(root, nullptr) << serialized;
74
+ const char* content = leptris_element_text(root);
75
+ ASSERT_NE(content, nullptr) << serialized;
76
+ EXPECT_STREQ(content, c.want_text) << c.xml;
77
+ if (std::string(c.xml).find("UK") != std::string::npos) {
78
+ // No <UK> element may have been minted.
79
+ EXPECT_EQ(leptris_element_first_child(root, "UK"), nullptr) << c.xml;
80
+ }
81
+
82
+ leptris_document_free(again);
83
+ leptris_free_string(serialized);
84
+ leptris_document_free(doc);
85
+ }
86
+ }
87
+
88
+ TEST(SerializeRoundTrip, KeepsEntityRefNodesUnescaped) {
89
+ // KEEP_ENTITY_REFS stores named references as ENTITY_REF nodes;
90
+ // those still serialize as &name; (they are markup, not text).
91
+ const char xml[] =
92
+ "<!DOCTYPE r [<!ENTITY foo \"x\">]><r>a &foo; b</r>";
93
+ LeptrisStatus st = LEPTRIS_OK;
94
+ LeptrisDocument doc = leptris_parse_string_flags(
95
+ xml, std::strlen(xml), LEPTRIS_PARSE_KEEP_ENTITY_REFS, &st);
96
+ ASSERT_NE(doc, nullptr);
97
+ char* serialized = leptris_document_serialize(doc, nullptr);
98
+ ASSERT_NE(serialized, nullptr);
99
+ EXPECT_NE(std::string(serialized).find("&foo;"), std::string::npos)
100
+ << serialized;
101
+ EXPECT_EQ(std::string(serialized).find("&amp;foo;"), std::string::npos)
102
+ << serialized;
103
+ leptris_free_string(serialized);
104
+ leptris_document_free(doc);
105
+ }
106
+
42
107
  TEST(SerializeRoundTrip, GrowsBufferForHugeTextContent) {
43
108
  // Regression for TODO 08: buffer_ensure_capacity must grow without
44
109
  // integer overflow or silent realloc-failure corruption.
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json",
3
3
  "name": "leptris",
4
- "version": "1.9.375",
4
+ "version": "1.9.376",
5
5
  "description": "High-performance XML parser and XPath 1.0 engine in C (libleptris)",
6
6
  "homepage": "https://github.com/leptris/leptris",
7
7
  "license": "MIT",
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: leptris
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.375.0
4
+ version: 1.9.376.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.