leptris 1.9.394.0-arm-linux → 1.9.396.0-arm-linux

data/vendor-src/libleptris/src/leptris/common/simd_text_neon.c

@@ -380,3 +380,116 @@ ptrdiff_t leptris_text_scan_run_neon(const char* s, size_t len, char term,
         return (p < end) ? (ptrdiff_t)(p - s) : -1;
     }
 }
+
+/* ---- UTF-8 validation (leptris#1842 A5) -------------------------------
+ * Same derivation as the SSE core in simd_text_avx2.c; NEON bytes are
+ * unsigned, so ranges are direct vcge/vcle pairs. */
+int leptris_utf8_valid_neon(const char* src, size_t len) {
+    const unsigned char* p = (const unsigned char*)src;
+    if (len < 16) return leptris_utf8_valid_scalar(src, len);
+
+    const uint8x16_t zero = vdupq_n_u8(0);
+    const uint8x16_t ff = vdupq_n_u8(0xFF);
+    const uint8x16_t b80 = vdupq_n_u8(0x80);
+    const uint8x16_t bBF = vdupq_n_u8(0xBF);
+    uint8x16_t prev = zero;
+
+#define IN_RANGE(v, lo, hi)                                             \
+    vandq_u8(vcgeq_u8((v), vdupq_n_u8(lo)), vcleq_u8((v), vdupq_n_u8(hi)))
+    size_t i = 0;
+    while (i + 16 <= len) {
+        uint8x16_t cur = vld1q_u8(p + i);
+        /* lane j = byte(j-1): vext(prev, cur, k) keeps the last k
+         * lanes of prev at the head -> k = 15/14/13. */
+        uint8x16_t b1 = vextq_u8(prev, cur, 15);
+        uint8x16_t b2 = vextq_u8(prev, cur, 14);
+        uint8x16_t b3 = vextq_u8(prev, cur, 13);
+
+        uint8x16_t st2c = IN_RANGE(cur, 0xC2, 0xDF);
+        uint8x16_t st3c = IN_RANGE(cur, 0xE0, 0xEF);
+        uint8x16_t st4c = IN_RANGE(cur, 0xF0, 0xF4);
+        uint8x16_t is_ascii = vcltq_u8(cur, b80);
+        uint8x16_t legal_start = vorrq_u8(st2c, vorrq_u8(st3c, st4c));
+
+        uint8x16_t st2b1 = IN_RANGE(b1, 0xC2, 0xDF);
+        uint8x16_t st3b1 = IN_RANGE(b1, 0xE0, 0xEF);
+        uint8x16_t st4b1 = IN_RANGE(b1, 0xF0, 0xF4);
+        uint8x16_t st3b2 = IN_RANGE(b2, 0xE0, 0xEF);
+        uint8x16_t st4b2 = IN_RANGE(b2, 0xF0, 0xF4);
+        uint8x16_t st4b3 = IN_RANGE(b3, 0xF0, 0xF4);
+
+        uint8x16_t must_cont =
+            vorrq_u8(vorrq_u8(st2b1, vorrq_u8(st3b1, st4b1)),
+                     vorrq_u8(vorrq_u8(st3b2, st4b2), st4b3));
+
+        const uint8x16_t bA0 = vdupq_n_u8(0xA0);
+        const uint8x16_t b90 = vdupq_n_u8(0x90);
+        const uint8x16_t b9F = vdupq_n_u8(0x9F);
+        const uint8x16_t b8F = vdupq_n_u8(0x8F);
+        uint8x16_t e0 = vceqq_u8(b1, vdupq_n_u8(0xE0));
+        uint8x16_t ed = vceqq_u8(b1, vdupq_n_u8(0xED));
+        uint8x16_t f0 = vceqq_u8(b1, vdupq_n_u8(0xF0));
+        uint8x16_t f4 = vceqq_u8(b1, vdupq_n_u8(0xF4));
+        uint8x16_t lo = vbslq_u8(e0, bA0, b80);
+        lo = vbslq_u8(f0, b90, lo);
+        uint8x16_t hi = vbslq_u8(ed, b9F, bBF);
+        hi = vbslq_u8(f4, b8F, hi);
+
+        uint8x16_t cont_ok = vandq_u8(vcgeq_u8(cur, lo), vcleq_u8(cur, hi));
+
+        uint8x16_t block_error = vorrq_u8(
+            vandq_u8(must_cont, veorq_u8(cont_ok, ff)),
+            vandq_u8(veorq_u8(must_cont, ff),
+                     vandq_u8(veorq_u8(is_ascii, ff),
+                              veorq_u8(legal_start, ff))));
+        /* any lane set? */
+        /* any lane set? */
+        if (vmaxvq_u8(block_error) != 0) return 0;
+        prev = cur;
+        i += 16;
+    }
+    if (i >= len) {
+        /* The input ENDS at the block edge: the last sequence must
+         * be complete (the scalar DFA's "len - i <= n" rule). Walk
+         * back over at most 3 continuations; s2 lands ON the last
+         * continuation (or the final non-continuation byte). */
+        size_t s2 = i;
+        size_t conts = 0;
+        while (s2 > 0 && (p[s2 - 1] & 0xC0) == 0x80 && conts < 3) {
+            s2--;
+            conts++;
+        }
+        if (conts > 0) {
+            if (s2 == 0) return 0; /* continuations, no opener anywhere */
+            unsigned op = p[s2 - 1]; /* the opener BEFORE the run */
+            unsigned owed = (op >= 0xF0) ? 3
+                          : (op >= 0xE0) ? 2
+                          : (op >= 0xC2) ? 1
+                                         : 0;
+            if (owed > conts) return 0;
+            return 1;
+        }
+        /* the last byte is not a continuation: it must be ASCII */
+        unsigned op = p[i - 1];
+        unsigned owed = (op >= 0xF0) ? 3
+                      : (op >= 0xE0) ? 2
+                      : (op >= 0xC2) ? 1
+                                     : 0;
+        return owed == 0 ? 1 : 0;
+    }
+    /* Tail: scalar DFA anchored at a character boundary (at most 3
+     * bytes back from the SIMD end; a sequence is at most 4) — the
+     * anchored run also covers a sequence spanning the block edge.
+     * One more step back when the byte before the anchor is a start
+     * byte: an opener in the final block lane reaches into the tail
+     * (400k-case randomized cross-check vs the scalar oracle). */
+    size_t start = i;
+    while (start > 0 && (p[start] & 0xC0) == 0x80 && i - start < 3)
+        start--;
+    if (i - start < 3 && start > 0) {
+        unsigned char b = p[start - 1];
+        if (b >= 0xC2 && b <= 0xF4) start--;
+    }
+    return leptris_utf8_valid_scalar((const char*)(p + start), len - start);
+}
+#undef IN_RANGE

data/vendor-src/libleptris/src/leptris/flat/direct_parse.c

@@ -2314,6 +2314,12 @@ static struct leptris_document* direct_parse_internal(char* buf, size_t len,
             if (*p.pos != '?') {
                 if (dp_nul(&p, p.pos)) goto fail;
                 p.pos++;
+                /* consuming the separator may exhaust the buffer
+                 * ("     <?xm" — the fuzzer shape): p.pos one past
+                 * p.end made p.end - p.pos underflow to SIZE_MAX and
+                 * the data memchr below walked off the allocation
+                 * (nightly-fuzz crash-aac3c9bb, no-LTO build) */
+                if (p.pos >= p.end) goto fail;
                 data_start = p.pos;
             }
             /* memchr for '?' — fast-skip PI data bodies. */
@@ -3531,48 +3537,15 @@ static int dp_declares_non_utf8(const char* s, size_t len) {
 }
 
 static int dp_utf8_valid(const char* s, size_t len) {
-    const unsigned char* p = (const unsigned char*)s;
-    size_t i = 0;
-    /* ASCII fast path: 8 bytes per iteration. The DFA below runs
-     * only when a high byte is present; ordinary documents pay one
-     * memory-bandwidth pass. */
-    while (i + 8 <= len) {
-        uint64_t w;
-        memcpy(&w, p + i, 8);
-        if (w & UINT64_C(0x8080808080808080)) break;
-        i += 8;
-    }
-    while (i < len && p[i] < 0x80) i++;
-    while (i < len) {
-        unsigned c = p[i];
-        if (c < 0x80) {
-            i++;
-            continue;
-        }
-        size_t n;
-        unsigned lo = 0x80;
-        unsigned hi = 0xBF;
-        if (c >= 0xC2 && c <= 0xDF) {
-            n = 1;
-        } else if (c >= 0xE0 && c <= 0xEF) {
-            n = 2;
-            if (c == 0xE0) lo = 0xA0;
-            else if (c == 0xED) hi = 0x9F;
-        } else if (c >= 0xF0 && c <= 0xF4) {
-            n = 3;
-            if (c == 0xF0) lo = 0x90;
-            else if (c == 0xF4) hi = 0x8F;
-        } else {
-            return 0;
-        }
-        if (len - i <= n) return 0;
-        if (p[i + 1] < lo || p[i + 1] > hi) return 0;
-        for (size_t k = 2; k <= n; k++) {
-            if (p[i + k] < 0x80 || p[i + k] > 0xBF) return 0;
-        }
-        i += n + 1;
-    }
-    return 1;
+    /* #1842 A5: the per-byte DFA moved to the SIMD layer —
+     * leptris_utf8_valid dispatches to boundary-vector cores (AVX2/
+     * NEON) with the DFA as the fallback+oracle. Pure accept/reject
+     * on the caller's bytes: no buffer-identity interaction (the A3
+     * revert settled that question for xml_buffer; this face only
+     * READS). */
+    (void)s;
+    (void)len;
+    return leptris_utf8_valid(s, len);
 }
 
 static int dp_input_encoding_ok(const char* s, size_t len) {
@@ -3658,7 +3631,10 @@ struct leptris_document* direct_parse_inplace(char* buf, size_t len) {
     if (!dp_declares_non_utf8(buf, len) && saw_high &&
         !dp_utf8_valid(buf, len))
         return NULL;
-    buf[len] = '\0';  /* Ensure NUL termination */
+    /* No sentinel write past len: the caller owns exactly `len`
+     * writable bytes (the header contract promises NULL terminators
+     * INSIDE the buffer; an exactly-sized caller trips ASAN, and
+     * bindings hand read-only views). The scanner takes `len`. */
     return direct_parse_internal(buf, len, 0, 0, 0, 0, 0, 0,
                                  lt_count, dq_count, sq_count);
 }

data/vendor-src/libleptris/src/leptris/xsd/compile.c

@@ -183,7 +183,9 @@ static void xsd_cm_free(XsdCm* cm) {
         free(cm->name);
         free(cm->type);
         free(cm->ns);
-        free(cm->ext_base);
+        free(cm->res_base);
+    free(cm->res_base_ns);
+    free(cm->ext_base);
         free(cm->ext_base_ns);
         free(cm->any_ns);
         free(cm->not_qname);
@@ -1858,11 +1860,11 @@ static void xsd_collect_decls_doc(LeptrisElement root, const char* eff_tns,
             xsd_collect_decls_doc(subroot, eff, sub_base, decls, n, max,
                                   claimed, nclaimed, cmax, depth + 1);
         }
-        /* leptris#1913: an xs:override body's declarations belong
-         * to the symbol space too — walk the override element as a
-         * pseudo-root (it has no include/import children of its
-         * own). */
-        if (strcmp(l, "override") == 0)
+        /* leptris#1913/#1926: override and redefine bodies'
+         * declarations belong to the symbol space too — walk the
+         * derivation element as a pseudo-root (it has no
+         * include/import children of its own). */
+        if (strcmp(l, "override") == 0 || strcmp(l, "redefine") == 0)
             xsd_collect_decls_doc(c, eff_tns, base_dir, decls, n, max,
                                   claimed, nclaimed, cmax, depth + 1);
         leptris_document_free(sub);
@@ -2343,14 +2345,20 @@ static void xsd_capture_root(struct leptris_xsd_schema* s,
                             if (sub_ns) free(sub_ns);
                             s->target_ns = saved_ns;
                         }
-                        /* leptris#1913 (XSD 1.1 §I.4): the override
-                         * element's OWN declaration children
-                         * replace same-named components of the base
-                         * just captured — capture them AFTER the
-                         * base so the lookup ladder (head-first)
-                         * sees the replacements. */
+                        /* leptris#1913 (XSD 1.1 §I.4) + #1926 (XSD
+                         * 1.0 §4.2.2): the override/redefine
+                         * element's OWN declaration children replace
+                         * same-named components of the base just
+                         * captured — capture them AFTER the base so
+                         * the lookup ladder (head-first) sees the
+                         * replacements. A redefine body that EXTENDS
+                         * the redefined name folds its extension
+                         * against the base at this moment (base
+                         * captured, ladder sees the new head), which
+                         * is exactly the redefine hook. */
                         if (subroot && same_ns &&
-                            strcmp(local, "override") == 0) {
+                            (strcmp(local, "override") == 0 ||
+                             strcmp(local, "redefine") == 0)) {
                             s->include_depth++;
                             s->capture_force = 1;
                             xsd_capture_root(s, (LeptrisElement)n);
@@ -2946,9 +2954,21 @@ static XsdCm* xsd_capture_complex_model(struct leptris_xsd_schema* s,
                             attrs = own_attrs;
                     }
                 } else {
-                    /* restriction: the derived model REPLACES */
+                    /* restriction: the derived model REPLACES.
+                     * leptris#1923: the base still rides the type —
+                     * libxml2 admits the restriction chain's declared
+                     * attributes at validation, so record the base
+                     * (local + resolved ns) for the strict-attribute
+                     * admission walk. */
                     root->first_child = xsd_cm_parse(s, dq, NULL);
                     attrs = xsd_capture_attr_rows(s, dq, 0);
+                    if (base && strncmp(base, "xs:", 3) != 0) {
+                        free(root->res_base);
+                        root->res_base = xsd_strdup(base);
+                        free(root->res_base_ns);
+                        root->res_base_ns =
+                            buri[0] ? xsd_strdup(buri) : NULL;
+                    }
                 }
                 /* XSD 1.1: asserts nested under the derivation */
                 if (s->version_11)
@@ -3015,6 +3035,16 @@ static XsdCm* xsd_capture_complex_model(struct leptris_xsd_schema* s,
                         } else {
                             root->text_type = xsd_strdup(base);
                         }
+                        /* leptris#1923: a simpleContent restriction
+                         * rides its base for strict-attribute
+                         * admission (libxml2 admits the chain's
+                         * declared attributes — ctE019). */
+                        if (strcmp(dl, "restriction") == 0) {
+                            free(root->res_base);
+                            root->res_base = xsd_strdup(base);
+                            free(root->res_base_ns);
+                            root->res_base_ns = NULL;
+                        }
                     }
                 }
                 attrs = xsd_capture_attr_rows(s, dq, 0);

data/vendor-src/libleptris/src/leptris/xsd/validate.c

@@ -635,7 +635,8 @@ static int any_attr_matches(const char* grammar, const char* uri,
 static void validate_attributes(struct xsd_validator* v,
                                 struct leptris_xsd_schema* s,
                                 LeptrisElement elem,
-                                const XsdTypeAttrs* ta) {
+                                const XsdTypeAttrs* ta,
+                                const XsdCm* chain_ct) {
     /* ta == NULL (no captured rows — attribute-less complexTypes)
      * skips the walk entirely; simple/builtin element validation
      * passes k_no_attrs to run the strict walk with zero declared
@@ -688,6 +689,37 @@ static void validate_attributes(struct xsd_validator* v,
             if (strcmp(a->name, local) == 0 ||
                 strcmp(a->name, qn) == 0)
                 declared = 1;
+        /* leptris#1923 (oracle parity with libxml2): the restriction
+         * chain's declared attributes stay visible for strict
+         * admission — a RESTRICTION that narrows the base's rows
+         * still admits what the base declared (ctE019). */
+        if (!declared && chain_ct) {
+            const XsdCm* anc = chain_ct;
+            for (int hops = 0; anc && anc->res_base && hops < 8;
+                 hops++) {
+                const XsdTypeAttrs* base_rows = xsd_find_type_attrs_ns(
+                    s, anc->res_base,
+                    anc->res_base_ns ? anc->res_base_ns
+                                     : xsd_target_ns(s));
+                if (!base_rows)
+                    base_rows = xsd_find_type_attrs(s, anc->res_base);
+                for (XsdAttrDecl* a = base_rows ? base_rows->attrs
+                                                : NULL;
+                     a && !declared; a = a->next)
+                    if (strcmp(a->name, local) == 0 ||
+                        strcmp(a->name, qn) == 0)
+                        declared = 1;
+                if (declared) break;
+                const XsdCm* base_ct = anc->res_base
+                    ? xsd_find_complex_ns(s, anc->res_base,
+                          anc->res_base_ns ? anc->res_base_ns
+                                           : xsd_target_ns(s))
+                    : NULL;
+                if (!base_ct)
+                    base_ct = xsd_find_complex(s, anc->res_base);
+                anc = base_ct;
+            }
+        }
         if (declared) continue;
         /* namespace of the attribute: engine-side attributes
          * carry their uri through the expanded form */
@@ -824,7 +856,7 @@ static void validate_local_uri(struct xsd_validator* v,
         const XsdTypeAttrs* rows =
             xsd_find_type_attrs_ns(s, type, euri);
         if (!rows) rows = xsd_find_type_attrs(s, type);
-        validate_attributes(v, s, elem, rows);
+        validate_attributes(v, s, elem, rows, ct);
         validate_children(v, s, elem, ct);
         /* XSD 1.1: the type's xs:assert list — context is the
          * element; evaluation errors are failures */
@@ -866,7 +898,7 @@ static void validate_local_uri(struct xsd_validator* v,
     if (strncmp(type, "xs:", 3) == 0 ||
         xsd_find_simple_pub(s, type)) {
         static const XsdTypeAttrs k_no_attrs;
-        validate_attributes(v, s, elem, &k_no_attrs);
+        validate_attributes(v, s, elem, &k_no_attrs, NULL);
         const char* text = leptris_element_text(elem);
         if (text && *text)
             text_valid(v, s, type, text, euri, "element", local,
@@ -886,7 +918,7 @@ static void validate_local_uri(struct xsd_validator* v,
                 (int)sizeof(builtin) &&
             xsd_builtin_known(builtin)) {
             static const XsdTypeAttrs k_no_attrs;
-            validate_attributes(v, s, elem, &k_no_attrs);
+            validate_attributes(v, s, elem, &k_no_attrs, NULL);
             const char* text = leptris_element_text(elem);
             if (text && *text)
                 text_valid(v, s, builtin, text, euri, "element",

data/vendor-src/libleptris/src/leptris/xsd/xsd_internal.h

@@ -120,6 +120,14 @@ typedef struct xsd_cm {
     char* ext_base; /* extension whose base was not yet captured at
                      * capture time (forward ref) — folded by the
                      * post-capture resolver (leptris-ruby#440 A) */
+    char* res_base; /* restriction whose base was not yet captured at
+                     * capture time — the post-capture resolver only
+                     * folds EXTENSION models; the restriction's model
+                     * replaces the base's, so this rides along solely
+                     * for strict-attribute ADMISSION (leptris#1923:
+                     * libxml2 admits the restriction chain's declared
+                     * attributes) */
+    char* res_base_ns;
     char* ext_base_ns; /* #1881: the base QName's RESOLVED namespace
                         * (prefix resolved against the extension
                         * element's in-scope declarations) — a set

data/vendor-src/libleptris/test/CMakeLists.txt

@@ -118,6 +118,7 @@ target_compile_definitions(test_sch_corpus PRIVATE
 leptris_add_test(test_parser     parser/test_parser.cpp)
 leptris_add_test(test_large_docs parser/test_large_documents.cpp)
 leptris_add_test(test_encoding   parser/test_encoding.cpp)
+leptris_add_test(test_utf8_simd  parser/test_utf8_simd.cpp)
 leptris_add_test(test_dom        dom/test_dom.cpp)
 leptris_add_test(test_element_children dom/test_element_children.cpp)
 leptris_add_test(test_entity_refs dom/test_entity_refs.cpp)

data/vendor-src/libleptris/test/parser/test_encoding.cpp

@@ -8,6 +8,8 @@
 
 #include "leptris.h"
 
+#include "../../src/leptris/common/simd_text.h"
+
 #include <cstring>
 #include <vector>
 
@@ -367,3 +369,116 @@ TEST(EncodingLatin1, WithEncodingConvertsToUtf8) {
     EXPECT_EQ((unsigned char)text[4], 0xA9);
     leptris_document_free(doc);
 }
+
+// leptris-py CI regression (2026-10-31): the binding's flow is
+// fast-path parse (raw UTF-16 into the UTF-8 parser) then retry via
+// the encoding-detecting entry. On x86 CI the retry of a BOM-only
+// document whose text contains a multi-byte character failed with
+// "malformed input". Mirror the flow stage by stage so the failing
+// stage is visible on every architecture.
+TEST(EncodingUtf16Le, PyBindingFlowBomWithMultibyteText) {
+    // "<?xml version='1.0'?><r><a>café</a></r>" as UTF-16LE + BOM —
+    // the exact bytes the py fixture produces.
+    static const unsigned char kDoc[] = {
+        0xFF, 0xFE,
+        '<', 0x00, '?', 0x00, 'x', 0x00, 'm', 0x00, 'l', 0x00,
+        ' ', 0x00, 'v', 0x00, 'e', 0x00, 'r', 0x00, 's', 0x00,
+        'i', 0x00, 'o', 0x00, 'n', 0x00, '=', 0x00, '\'', 0x00,
+        '1', 0x00, '.', 0x00, '0', 0x00, '\'', 0x00, '?', 0x00,
+        '>', 0x00,
+        '<', 0x00, 'r', 0x00, '>', 0x00,
+        '<', 0x00, 'a', 0x00, '>', 0x00,
+        'c', 0x00, 'a', 0x00, 'f', 0x00, 0xE9, 0x00,
+        '<', 0x00, '/', 0x00, 'a', 0x00, '>', 0x00,
+        '<', 0x00, '/', 0x00, 'r', 0x00, '>', 0x00,
+    };
+    const size_t n = sizeof kDoc;
+    ASSERT_EQ(n, 80u);
+
+    // Stage 1 — the UTF-8 entry over raw UTF-16 (py fast path): the
+    // lenient contract is parse-or-fail, never a crash. The inplace
+    // entry owns the buffer it is given (NUL-masking is part of the
+    // inplace contract), so the scratch copy absorbs the mutation —
+    // the pristine kDoc stays for the retry.
+    std::vector<unsigned char> buf(kDoc, kDoc + n);
+    LeptrisStatus st = LEPTRIS_OK;
+    LeptrisDocument fast = leptris_parse_string_inplace(
+        reinterpret_cast<char*>(buf.data()), n, &st);
+    if (fast) leptris_document_free(fast);
+    ASSERT_EQ(buf.size(), n) << "inplace parse must not resize";
+
+    // Stage 2 — the encoding-detecting retry on the pristine bytes:
+    // BOM-only, multi-byte payload; must convert and parse.
+    LeptrisDocument doc = leptris_parse_string_with_encoding(
+        reinterpret_cast<const char*>(kDoc), n, &st);
+    ASSERT_NE(doc, nullptr) << "status: " << leptris_status_string(st);
+
+    LeptrisElement root = leptris_document_root(doc);
+    ASSERT_NE(root, nullptr);
+    EXPECT_STREQ(leptris_element_name(root), "r");
+    LeptrisElement a = leptris_element_first_child_any(root);
+    ASSERT_NE(a, nullptr);
+    const char* text = leptris_element_text(a);
+    ASSERT_STREQ(text, "caf\xC3\xA9");
+
+    leptris_document_free(doc);
+}
+
+// x86-only multibyte-parse bisection (follows the PyBindingFlow spec):
+// the Utf8Simd matrix is green on every lane while multibyte PARSES
+// fail on all four x86 lanes. Stage the flow so the failing surface
+// names itself: validator verdict, per-sequence-length parses, the
+// converted-buffer shape, and the interleaved vs default lane.
+TEST(EncodingUtf8Strict, X86MultibyteBisection) {
+    const char two[] = "<r>caf\xC3\xA9</r>";
+    const char three[] = "<r>\xE4\xB8\xAD</r>";
+    const char four[] = "<r>\xF0\x9F\x98\x80</r>";
+    const char mix[] = "<r>caf\xC3\xA9 \xE4\xB8\xAD \xF0\x9F\x98\x80</r>";
+    EXPECT_EQ(leptris_utf8_valid(two, strlen(two)), 1) << "validator: 2-byte doc";
+    EXPECT_EQ(leptris_utf8_valid(three, strlen(three)), 1) << "validator: 3-byte doc";
+    EXPECT_EQ(leptris_utf8_valid(four, strlen(four)), 1) << "validator: 4-byte doc";
+    /* Localize the offending lane: every prefix of the mixed doc is
+     * either valid or ends mid-sequence (which the scalar oracle
+     * rejects too) — print the verdict map so the first SIMD/scalar
+     * divergence names its byte offset. */
+    {
+        std::string map;
+        for (size_t k = 0; k <= strlen(mix); k++) {
+            int sc = leptris_utf8_valid_scalar(mix, k);
+            int si = leptris_utf8_valid(mix, k);
+            if (sc != si || k == strlen(mix))
+                map += " [" + std::to_string(k) + "]:" + std::to_string(sc)
+                     + "/" + std::to_string(si);
+        }
+        EXPECT_EQ(leptris_utf8_valid(mix, strlen(mix)), 1)
+            << "validator: mixed doc; divergences scalar/simd" << map;
+    }
+
+    LeptrisStatus st = LEPTRIS_OK;
+    LeptrisDocument d2 = leptris_parse_string(two, strlen(two), &st);
+    EXPECT_NE(d2, nullptr) << "parse 2-byte: " << leptris_status_string(st);
+    if (d2) leptris_document_free(d2);
+    LeptrisDocument d3 = leptris_parse_string(three, strlen(three), &st);
+    EXPECT_NE(d3, nullptr) << "parse 3-byte: " << leptris_status_string(st);
+    if (d3) leptris_document_free(d3);
+    LeptrisDocument d4 = leptris_parse_string(four, strlen(four), &st);
+    EXPECT_NE(d4, nullptr) << "parse 4-byte: " << leptris_status_string(st);
+    if (d4) leptris_document_free(d4);
+
+    /* The converted-buffer shape from the py flow (40 bytes). */
+    const char conv[] = "<?xml version='1.0'?><r><a>caf\xC3\xA9</a></r>";
+    EXPECT_EQ(leptris_utf8_valid(conv, strlen(conv)), 1) << "validator: converted buffer";
+    LeptrisDocument dc = leptris_parse_string(conv, strlen(conv), &st);
+    EXPECT_NE(dc, nullptr) << "parse converted: " << leptris_status_string(st);
+    if (dc) leptris_document_free(dc);
+
+    /* The default (non-interleaved) lane via RECOVER. */
+    LeptrisParseOptions opts;
+    memset(&opts, 0, sizeof opts);
+    opts.flags = LEPTRIS_PARSE_RECOVER;
+    opts.strict_mode = -1;
+    LeptrisDocument dr = leptris_parse_string_ex(mix, strlen(mix),
+        &opts, &st);
+    EXPECT_NE(dr, nullptr) << "parse mixed (default lane): " << leptris_status_string(st);
+    if (dr) leptris_document_free(dr);
+}

data/vendor-src/libleptris/test/parser/test_utf8_simd.cpp

@@ -0,0 +1,130 @@
+// #1842 A5: the SIMD UTF-8 validator must agree with the scalar DFA
+// on every input — adversarial boundary matrix plus a randomized
+// cross-check over lengths straddling the 16-byte block edge.
+#include "../../src/leptris/common/simd_text.h"
+
+#include <gtest/gtest.h>
+#include <cstring>
+#include <cstdlib>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+}
+
+TEST(Utf8Simd, SingleByteValues) {
+    for (int b = 0; b <= 255; b++) {
+        char buf[1] = {(char)b};
+        int scalar = leptris_utf8_valid_scalar(buf, 1);
+        int simd = leptris_utf8_valid(buf, 1);
+        EXPECT_EQ(simd, scalar) << "byte 0x" << std::hex << b << std::dec;
+        bool expect = (b < 0x80); /* a lone start byte is truncated */
+        EXPECT_EQ(scalar, expect ? 1 : 0)
+            << "byte 0x" << std::hex << b << std::dec;
+    }
+}
+
+TEST(Utf8Simd, SequenceMatrixAcrossBlockEdges) {
+    // (case, expected) — corrupt rows must be rejected too.
+    const std::vector<std::pair<std::string, int>> cases = {
+        // valid
+        {{"\xC2\xA0"}, 1}, {{"\xDF\xBF"}, 1}, {{"\xE0\xA0\x80"}, 1},
+        {{"\xE1\x80\x80"}, 1}, {{"\xED\x9F\xBF"}, 1}, {{"\xEE\x80\x80"}, 1},
+        {{"\xEF\xBF\xBD"}, 1}, {{"\xF0\x90\x80\x80"}, 1},
+        {{"\xF1\xBF\xBF\xBF"}, 1}, {{"\xF3\xA0\x80\x80"}, 1},
+        {{"\xF4\x8F\xBF\xBF"}, 1},
+        // overlong / surrogates / out-of-range
+        {{"\xC0\x80"}, 0}, {{"\xC1\xBF"}, 0}, {{"\xE0\x80\x80"}, 0},
+        {{"\xE0\x9F\xBF"}, 0}, {{"\xED\xA0\x80"}, 0}, {{"\xED\xBF\xBF"}, 0},
+        {{"\xF0\x80\x80\x80"}, 0}, {{"\xF0\x8F\xBF\xBF"}, 0},
+        {{"\xF4\x90\x80\x80"}, 0}, {{"\xF5\x80\x80\x80"}, 0},
+        // truncated (each prefix)
+        {{"\xC2"}, 0}, {{"\xE0\xA0"}, 0}, {{"\xF0\x90\x80"}, 0},
+        {{"\xF4\x8F"}, 0},
+        // stray continuation
+        {{"\x80"}, 0}, {{"a\x80" "b"}, 0}, {{"\xBF"}, 0},
+    };
+    for (std::vector<std::pair<std::string, int> >::const_iterator
+             it = cases.begin();
+         it != cases.end(); ++it) {
+        const std::string& s = it->first;
+        const int expect = it->second;
+        for (int pad = 0; pad < 20; pad++) {
+            std::string buf((size_t)pad, 'a');
+            buf += s;
+            int scalar =
+                leptris_utf8_valid_scalar(buf.data(), buf.size());
+            int simd = leptris_utf8_valid(buf.data(), buf.size());
+            EXPECT_EQ(simd, scalar)
+                << "case len=" << s.size() << " pad=" << pad;
+            EXPECT_EQ(scalar, expect)
+                << "verdict drifted: len=" << s.size() << " pad=" << pad;
+        }
+    }
+}
+
+TEST(Utf8Simd, BlockEdgeSizes) {
+    for (size_t n = 0; n <= 40; n++) {
+        std::string buf(n, 'x');
+        buf += "\xC3\xA9";
+        ASSERT_EQ(leptris_utf8_valid_scalar(buf.data(), buf.size()), 1);
+        ASSERT_EQ(leptris_utf8_valid(buf.data(), buf.size()), 1) << "n=" << n;
+    }
+    for (size_t n = 0; n <= 40; n++) {
+        std::string buf(n, 'x');
+        buf += "\xF0\x90\x80";
+        ASSERT_EQ(leptris_utf8_valid_scalar(buf.data(), buf.size()), 0);
+        ASSERT_EQ(leptris_utf8_valid(buf.data(), buf.size()), 0) << "n=" << n;
+    }
+}
+
+TEST(Utf8Simd, RandomizedCrossCheck) {
+    unsigned seed = 0x1842;
+    auto rnd = [&seed]() {
+        seed = seed * 1103515245u + 12345u;
+        return (seed >> 16) & 0xFF;
+    };
+    for (int trial = 0; trial < 4000; trial++) {
+        size_t len = (size_t)(rnd() % 97);
+        std::string buf;
+        for (size_t i = 0; i < len; i++) {
+            unsigned r = rnd() % 100;
+            char c;
+            if (r < 55) c = (char)('a' + (r % 26));
+            else if (r < 75) c = (char)(0xC0 + (r % 0x40));
+            else c = (char)(0x80 + (r % 0x80));
+            buf.push_back(c);
+        }
+        /* The plain corpus rarely produces the boundary shapes that
+         * broke the cores (x86-TSAN 2026-10-31): a start byte in the
+         * final block lane with a non-continuation after it, and
+         * sequences straddling 16/32-byte edges. Append an ASCII
+         * tail after a valid-or-not multibyte run so every trial
+         * also probes the tail-anchor at a shifted offset. */
+        if (trial % 3 == 0) {
+            static const char* mids[] = {
+                "\xC3\xA9", "\xE4\xB8\xAD", "\xF0\x9F\x98\x80",
+                "\xC3", "\xE0\xA0", "\xF0\x90\x80", "\xED\xA0\x80",
+                "\xF4\x8F\xBF\xBF", "\xC2\xBF", "\xEF\xBF\xBD"
+            };
+            buf += mids[trial % 10];
+            buf += "zz";
+        }
+        int scalar = leptris_utf8_valid_scalar(buf.data(), buf.size());
+        int simd = leptris_utf8_valid(buf.data(), buf.size());
+        EXPECT_EQ(simd, scalar)
+            << "trial " << trial << " len " << buf.size();
+    }
+}
+
+TEST(Utf8Simd, DenseValidDocument) {
+    std::string buf;
+    buf.reserve(64 * 1024);
+    const char* unit = "héllo wörld — 日本語テキスト 🎉 emoji §naïve";
+    for (int i = 0; i < 1024; i++) buf += unit;
+    EXPECT_EQ(leptris_utf8_valid_scalar(buf.data(), buf.size()), 1);
+    EXPECT_EQ(leptris_utf8_valid(buf.data(), buf.size()), 1);
+    buf[buf.size() / 2] = (char)0xF5;
+    EXPECT_EQ(leptris_utf8_valid(buf.data(), buf.size()), 0);
+}