leptris 1.9.391.0-arm-linux → 1.9.396.0-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ad8a3795b30020ae925fb64a19f9acadbcd26ab10da9283a145be14f6e2dede
-  data.tar.gz: 3838a8f9937409fefcce230eed59d0c6e46c53e069159e00722538ebfb695193
+  metadata.gz: d9946e2b50f4e9284001bb923683803b7cca891c28154e28129c4a079a6ace83
+  data.tar.gz: 1da14aa052d16d328f2c49ac9a59f023b8ee192324b0d8e510402230e1e8866e
 SHA512:
-  metadata.gz: d2272eec63c0c2ddd3dab229cc209255b9ac8b4056659985f1195d0e00b175abfa98ffb1ac59f66b2a286ea6293edf201816c7201eac74dfd0ae0d3fcebcee54
-  data.tar.gz: 45f7f2a9af016191fb590dd2c110e150564567799a6e76c62513ffee715e2c78b6d119190e2c4bb36fc697d96b345d59977c33102b25f082dfad4ef21626316b
+  metadata.gz: 89551eed859940f012e3e49b276c0f4b998597a803bb2cfab1eb73d2c7787ccd753a3040946a410bd1fc2b8fe00b436fecaa5a37012defe57994a783861dba9e
+  data.tar.gz: a775d3e6462e2b1dea2eaabcc21239b7a6d0b6df02e9a10020033d5b03d0f34141ee68ef49bdb4355847ea65891cbd13bb93896d0ca6d948f85f677203ac8fac

data/Rakefile

@@ -8,7 +8,7 @@ RSpec::Core::RakeTask.new(:spec)
 # Pin for `rake compile` and the platform-gem builds. Keep in lockstep
 # with .github/workflows/build.yml (which calls `rake compile`) and the
 # CHANGELOG when libleptris releases.
-LIBLEPTRIS_VERSION = "1.9.391"
+LIBLEPTRIS_VERSION = "1.9.396"
 # Vendored alongside libleptris for fn:normalize-unicode (TODO
 # .restructure/20): built per platform with a RELOCATABLE @rpath
 # install name, loaded by ffi.rb before libleptris so the
@@ -401,8 +401,14 @@ namespace :audit do
       .map { |n| n.sub(/\A_/, "") }
       .select { |n| n.start_with?("leptris_") }
       .map { |n| n.sub(/\Aleptris_/, "") }.sort
-    attached = File.read("lib/leptris/xml/ffi.rb")
-      .scan(/attach_function :leptris_([a-z_0-9]+)/).flatten.sort
+    attached = (
+      File.read("lib/leptris/xml/ffi.rb")
+        .scan(/attach_function :leptris_([a-z_0-9]+)/).flatten +
+      # The native shim's C->C dlsym dispatches (#572: struct-value
+      # returns libffi can't carry) are bound surface too.
+      File.read("ext/leptris/native/native.c")
+        .scan(/lib_sym\((?:h|handle), "leptris_([a-z_0-9]+)"\)/).flatten
+    ).sort
     # Version-gated faces: soft-attached because engines BEFORE the
     # named release lack the symbol (the load must survive them).
     # They join the audit only when the engine pin reaches that

data/ext/leptris/native/native.c

@@ -278,6 +278,14 @@ static void *lib_open(const char *path) { return dlopen(path, RTLD_NOW); }
 static void *lib_sym(void *h, const char *name) { return dlsym(h, name); }
 #endif
 
+/* leptris-ruby#572: the engine's capability descriptor (1.9.392+). */
+typedef struct {
+    unsigned long long caps;
+    unsigned descriptor_version;
+} rb_caps_t;
+typedef rb_caps_t (*capabilities_fn)(void);
+static capabilities_fn f_capabilities;
+
 static void resolve_symbols(const char *lib_path)
 {
     void *h = lib_open(lib_path);
@@ -314,6 +322,7 @@ static void resolve_symbols(const char *lib_path)
     /* Single-crossing element construction (libleptris 1.9.237,
      * #1344): NULL in older engine pins; the create path falls
      * back to create + per-pair sets. */
+    f_capabilities = (capabilities_fn)lib_sym(h, "leptris_capabilities");
     f_elem_new_with_attrs =
         (elem_new_attrs_fn)lib_sym(h, "leptris_element_new_with_attributes");
     f_text_create = (text_create_fn)lib_sym(h, "leptris_text_node_create");
@@ -3965,6 +3974,25 @@ static VALUE nf_sax_parser_free(VALUE self, VALUE obj)
     return Qnil;
 }
 
+/* leptris-ruby#572: the engine's capability descriptor as a plain
+ * hash — libffi's small-struct value return proved unreliable across
+ * ABIs, so the shim reads the C struct and returns the two fields
+ * (raw bitset + descriptor version; bits append per #1905). */
+static VALUE nf_capabilities(VALUE module) {
+    VALUE hash = rb_hash_new();
+    if (!f_capabilities) {
+        rb_hash_aset(hash, ID2SYM(rb_intern("caps")), Qnil);
+        rb_hash_aset(hash, ID2SYM(rb_intern("descriptor_version")), Qnil);
+        return hash;
+    }
+    rb_caps_t c = f_capabilities();
+    rb_hash_aset(hash, ID2SYM(rb_intern("caps")),
+                 ULL2NUM(c.caps));
+    rb_hash_aset(hash, ID2SYM(rb_intern("descriptor_version")),
+                 UINT2NUM(c.descriptor_version));
+    return hash;
+}
+
 LEPTRIS_INIT_EXPORT void Init_native(void)
 {
     VALUE m_leptris, m_xml, m_native;
@@ -4084,6 +4112,8 @@ LEPTRIS_INIT_EXPORT void Init_native(void)
     c_sax_parser = rb_define_class_under(m_native, "SaxParser",
                                          c_object);
     rb_define_module_function(m_native, "sax_parse", nf_sax_parse, 3);
+    rb_define_module_function(m_native, "capabilities",
+                              nf_capabilities, 0);
     rb_define_module_function(m_native, "sax_parser_create",
                               nf_sax_parser_create, 2);
     rb_define_module_function(m_native, "sax_parser_feed",

data/lib/leptris/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Leptris
-  VERSION = "1.9.391.0"
+  VERSION = "1.9.396.0"
 end

data/lib/leptris/xml/ffi.rb

@@ -566,6 +566,11 @@ module Leptris
         [:string, :size_t, :leptris_plan, :pointer],
         :leptris_plan_result
 
+      # leptris-ruby#572: the engine's capability descriptor is read
+      # through the native shim's Native.capabilities (libffi's
+      # small-struct value return proved ABI-unreliable); the shim
+      # exposes the raw bitset + descriptor version and re-encodes
+      # nothing (bits append per contract).
       attach_function :leptris_version, [], UTF8_STRING
       attach_function :leptris_version_components, [:pointer, :pointer, :pointer], :void
 

data/lib/leptris/xml.rb

@@ -43,6 +43,31 @@ module Leptris
 
     # Nokogiri-style top-level entry points (Nokogiri::XML(...) /
     # Nokogiri::XML.parse). Delegates to Document.parse.
+    # leptris-ruby#572: the engine's capability descriptor — the
+    # raw bitset plus descriptor version (leptris/leptris#1905).
+    # Frozen and memoized: the engine's answers are fixed per binary.
+    # Consumers (moxml#504) OR this with wrapper knowledge; bits
+    # append per the engine's contract — never reorder.
+    def self.capabilities
+      @capabilities ||=
+        begin
+          raw =
+            begin
+              Native.capabilities
+            rescue NameError
+              # the native layer loads with `require "leptris"`;
+              # a direct `require "leptris/xml"` has not loaded it
+              # yet — the explicit entry is the parallel-API door.
+              require "leptris"
+              Native.capabilities
+            end
+          {
+            caps: raw[:caps],
+            descriptor_version: raw[:descriptor_version]
+          }.freeze
+        end
+    end
+
     def self.parse(xml_or_io, options: nil, readonly: false,
                    recover: true, strict: false)
       recover = false if strict

data/vendor-src/README.md

@@ -1,7 +1,7 @@
 Sources vendored in this gem (recompile rights; the ruby
 variant builds them at install via extconf.rb):
 
-- libleptris v1.9.391
+- libleptris v1.9.396
 - utf8proc v2.11.0
 
 Rebuild by hand:

data/vendor-src/libleptris/CHANGELOG.md

@@ -1,5 +1,151 @@
 ## [Unreleased]
 
+## [1.9.396] - 2026-10-31
+
+### Fixed
+
+- **The AVX2 UTF-8 validator rejected valid multi-byte input
+  (leptris#1842 A5 follow-up).** Three defects shipped with the
+  1.9.395 SIMD gate, all invisible on arm64: the ascii lane's
+  polarity was inverted (`cmplt(cur, zero)` is the NOT-ascii half,
+  so every ASCII byte tripped the error term), both
+  continuation-range bounds compared in the wrong direction
+  (`cont_ok` was unsatisfiable — NEON's direct `vcge/vcle` pair
+  became a swapped saturating-subtraction idiom), and the tail
+  anchor walked back only over continuation runs, letting an
+  opener in the final block lane escape validation (both cores).
+  The leptris-py UTF-16 BOM lane caught it (`malformed input` on
+  `café`); x86 CI failed 25+ tests. Now verified by a 400k-case
+  lane-math simulation against the scalar oracle plus boundary
+  shapes seeded into the randomized cross-check.
+
+- **`leptris_parse_string_inplace` no longer writes past the
+  buffer.** The NUL sentinel at `buf[len]` exceeded the documented
+  contract (terminators INSIDE the buffer); an exactly-sized
+  caller tripped ASAN. The scanner takes `len` and never read it.
+
+- **`xs:redefine` bodies were dropped (leptris#1926).** A
+  `redefine`d complexType whose body declared new particles (or
+  attributes) lost them: the extension derived from the base type
+  captured its model against the BASE body, and the redefinition was
+  never folded in. Bodies are now captured after the base resolves —
+  the same mechanism `xs:override` already uses (PR #1913) — so the
+  ladder resolves head-first through the redefinition and the
+  extension folds against the redefined content. `xs:group`/attr
+  refs inside redefine bodies are collected like override bodies.
+
+
+
+## [1.9.395] - 2026-10-31
+
+### Added
+
+- **Boundary-vector UTF-8 validation (leptris#1842 A5, PRs #1922/#1924).**
+  `leptris_utf8_valid` — AVX2/NEON cores over the existing dispatch
+  layer, the scalar DFA as the fallback and oracle — pinned by an
+  adversarial matrix plus a 4000-case randomized cross-check. The
+  parser's UTF-8 gate now routes through it: **parse of UTF-8-dense
+  documents is +32%** (3.70ms -> 2.81ms on a 2.2MB catalog; the
+  validator alone ~3.7 GB/s on NEON). Unblocked line-level profiling
+  first: the no-LTO `-g` build resolves file:line through
+  dsymutil/atos (ThinLTO bitcode was the blocker).
+
+### Fixed
+
+- **PI data-scan bounds (fuzzer class).** Consuming the PI target's
+  separator could leave `p.pos` one past `p.end`; `p.end - p.pos`
+  underflowed to SIZE_MAX and the data `memchr` walked off the
+  allocation (the nightly-fuzz witness segfaulted under the no-LTO
+  build — allocator luck hid it on LTO).
+- **ctE019 regression (leptris#1923, PR #1927).** A chameleon-included
+  attribute referenced through a restriction base was false-rejected
+  as "not declared (strict)" — the 1.9.391 pending-ref fix made the
+  forward-referenced rows visible, surfacing that only the derived
+  type's rows were consulted. The oracle (xmllint) validates ctE019:
+  both derivation spellings now record the restriction base and the
+  strict-attribute admission walks the chain (admission only;
+  required/fixed/type checks stay derived-only).
+- drop the const-path pristine carve (leptris#1842 A3) (parse)
+
+
+
+## [1.9.394] - 2026-10-30
+
+### Fixed
+
+- **xs:override loads its base and replaces the overridden components
+  (leptris#1913, PR #1917).** `override` was absent from the
+  declaration gate and the include-family chase, so the base document
+  never loaded and the override body's nested declarations were never
+  captured — the compiled schema had an empty symbol space (decls=0,
+  every instance element "not declared"). Override now chases its
+  schemaLocation like include (same targetNamespace), then captures
+  its own declaration children as a pseudo-root AFTER the base so the
+  head-first lookup ladder sees the replacements; the attributeGroup
+  pre-pass honors a capture-force flag in that body, and the ref-check
+  declaration collector walks the override body too. Witness: base
+  `Doc` replaced by the override's `Doc` — `<doc><b>1</b></doc>`
+  validates, `<doc><a>x</a></doc>` is rejected.
+
+
+
+## [1.9.393] - 2026-10-30
+
+### Added
+
+- **Capability bit for entity-reference nodes (leptris#1904, PR #1915).**
+  `LEPTRIS_PARSE_KEEP_ENTITY_REFS` already parses references as
+  first-class `LEPTRIS_NODE_TYPE_ENTITY_REF` nodes — text segments
+  split around the five built-ins and custom named references
+  (numeric character references stay text, XML 1.0 §4.1) — but
+  nothing advertised it. `LEPTRIS_CAP_ENTITY_REF_NODES` lets a
+  binding probe once and parse with the flag instead of re-scanning
+  expanded text; a spec pins the DOM shape and the
+  string-value / serialization / XPath agreement.
+
+### Fixed
+
+- **XSD 1.1 was inert for every `vc:minVersion`-marked schema
+  (leptris#1910, leptris#1911, PR #1914).** The version detector
+  read only the root's plain `version` attribute; the W3C suite and
+  the binding ecosystem mark 1.1 with `vc:minVersion` in the
+  versioning namespace. With the capture gates off,
+  `xs:alternative` and `xs:assert` compiled to nothing —
+  conditional type assignment applied the declared base type's
+  model in both directions (false rejects and false accepts), and
+  asserts never attached to their types. The detector now honors
+  `version`, `vc:minVersion`, and a `vc:maxVersion="1.0"` pin, and
+  `xs:openContent` interleave/suffix semantics engage with it
+  (leptris#1912's repro validates clean).
+- **Value-comparison keywords accept name-test operands
+  (`w le h`).** The XPath `eq/ne/lt/le/gt/ge` operators only read
+  as operators when the right operand opened with a scalar, paren,
+  variable, or function call — an `xs:assert test="w le h"` failed
+  to compile. Adjacent name tokens are never a valid path
+  continuation, so a bare name-test follower is unambiguously an
+  operand; every path start is now admitted.
+
+
+
+## [1.9.392] - 2026-10-30
+
+### Added
+
+- **Capability descriptor face (leptris#1905, PR #1908).**
+  `leptris_capabilities()` returns a versioned bitset of every face
+  this installed engine serves — the engine-side half of the
+  moxml#504 capability report. Bindings OR it with their own wrapper
+  knowledge instead of maintaining scattered version-gate
+  constants; build-conditional faces (DUCET collation, utf8proc,
+  iconv, AVX2) follow the same `LEPTRIS_HAS_*` macros that gate the
+  sources, so the descriptor cannot drift from the binary it ships
+  in. Contract: bits append, never reorder; the descriptor version
+  gates shape changes. The spec anchors every unconditional bit to
+  its real face symbol — a bit without a linked face fails the
+  build, not the assertion.
+
+
+
 ## [1.9.391] - 2026-10-30
 
 ### Fixed

data/vendor-src/libleptris/CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(leptris
-    VERSION 1.9.391
+    VERSION 1.9.396
     DESCRIPTION "Fast XML parser and XPath evaluator in pure C"
     LANGUAGES C CXX
 )

data/vendor-src/libleptris/README.adoc

@@ -744,6 +744,43 @@ To pin enum values (bindings hard-code these):
 ctest --test-dir build -R HeaderHygiene
 ----
 
+=== Capability descriptor (one call: what does this engine serve?)
+
+Bindings and embedders stop maintaining scattered version-gate
+constants: one call returns a versioned bitset of every face this
+installed engine serves. Build-conditional faces (Unicode
+normalization, iconv, DUCET collation, AVX2) answer honestly for the
+exact library you linked. New capabilities APPEND bits — the
+descriptor version gates shape changes.
+
+[source,c]
+----
+LeptrisCapabilities c = leptris_capabilities();
+if (c.descriptor_version >= 1 && (c.caps & LEPTRIS_CAP_XSD)) {
+    /* xsd compile/validate faces are present */
+}
+----
+
+Entity references: parsing with `LEPTRIS_PARSE_KEEP_ENTITY_REFS`
+yields `LEPTRIS_NODE_TYPE_ENTITY_REF` nodes instead of pre-expanded
+text — segments split around the five built-ins and custom named
+references (numeric character references stay text, per XML 1.0
+§4.1). String-value, serialization, and XPath all see the expanded
+value. Probe `LEPTRIS_CAP_ENTITY_REF_NODES` before relying on the
+mode.
+
+The bits: `LEPTRIS_CAP_PARSE`, `LEPTRIS_CAP_DOM`, `LEPTRIS_CAP_XPATH`,
+`LEPTRIS_CAP_XQUERY`, `LEPTRIS_CAP_DTD`, `LEPTRIS_CAP_SAX`,
+`LEPTRIS_CAP_HTML`, `LEPTRIS_CAP_XINCLUDE`, `LEPTRIS_CAP_DIFF`,
+`LEPTRIS_CAP_DIGEST`, `LEPTRIS_CAP_XSD`, `LEPTRIS_CAP_XSD_SET`,
+`LEPTRIS_CAP_RELAXNG`, `LEPTRIS_CAP_SCHEMATRON`, `LEPTRIS_CAP_XSLT`,
+`LEPTRIS_CAP_DESCRIPTOR`, `LEPTRIS_CAP_NS_SNAPSHOT_BULK`,
+`LEPTRIS_CAP_CHILDREN_BATCH`, `LEPTRIS_CAP_NAME_COMPARE`,
+`LEPTRIS_CAP_VERSION`, and the build-conditional
+`LEPTRIS_CAP_COLLATION_DUCET`, `LEPTRIS_CAP_UNICODE_UTF8PROC`,
+`LEPTRIS_CAP_ENCODING_ICONV`, `LEPTRIS_CAP_SIMD_AVX2`,
+`LEPTRIS_CAP_ENTITY_REF_NODES`.
+
 [#FFI]
 == FFI design
 

data/vendor-src/libleptris/src/CMakeLists.txt

@@ -300,16 +300,23 @@ target_compile_definitions(leptris_objects PUBLIC _POSIX_C_SOURCE=200809L)
 # them all into the same library, dispatch at runtime via cpu.c. The
 # scalar TU (simd_text.c) is always built; these add accelerated paths.
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|AMD64|i[3-6]86)")
-    check_c_compiler_flag("-mavx2" LEPTRIS_HAVE_AVX2_FLAG)
+    # MSVC has no -mavx2; /arch:AVX2 enables the whole AVX set for the
+    # TU (the windows lane shipped scalar-only and the capability bit
+    # disagreed with test_capabilities — 2026-10-31).
+    if(MSVC)
+        set(LEPTRIS_AVX2_FLAG "/arch:AVX2")
+    else()
+        set(LEPTRIS_AVX2_FLAG "-mavx2")
+    endif()
+    check_c_compiler_flag("${LEPTRIS_AVX2_FLAG}" LEPTRIS_HAVE_AVX2_FLAG)
     if(LEPTRIS_HAVE_AVX2_FLAG)
         target_sources(leptris_objects PRIVATE leptris/common/simd_text_avx2.c)
         set_source_files_properties(leptris/common/simd_text_avx2.c
-            PROPERTIES COMPILE_FLAGS "-mavx2")
+            PROPERTIES COMPILE_FLAGS "${LEPTRIS_AVX2_FLAG}")
         # Consumers (simd_text.c dispatch) reference the AVX2 symbol only
-        # when this define is present — MSVC rejects -mavx2 (uses /arch:),
-        # so the TU is skipped there and the dispatch falls back to scalar.
+        # when this define is present.
         target_compile_definitions(leptris_objects PRIVATE LEPTRIS_HAS_AVX2_BUILD=1)
-        message(STATUS "AOT SIMD: AVX2 text scan ENABLED")
+        message(STATUS "AOT SIMD: AVX2 text scan ENABLED (${LEPTRIS_AVX2_FLAG})")
     else()
         message(STATUS "AOT SIMD: AVX2 unavailable (flag rejected)")
     endif()

data/vendor-src/libleptris/src/include/leptris/types.h

@@ -447,6 +447,58 @@ typedef struct leptris_xpath_compiled* LeptrisXPathCompiled;
  * compiled — apply it to any number of documents, from any threads. */
 typedef struct leptris_xslt* LeptrisXslt;
 
+/* ============================================================================
+ * Capability Descriptor (leptris#1905)
+ * ============================================================================ */
+
+/* What this INSTALLED engine serves — one call, one versioned
+ * bitset. Contract: new capabilities APPEND bits; a bit is never
+ * removed, repurposed, or reordered — that is a descriptor-version
+ * bump. Build-conditional faces (utf8proc/iconv/DUCET/AVX2) answer
+ * honestly for the exact library a consumer linked. Bindings OR
+ * this with their own wrapper knowledge; consumers stop maintaining
+ * scattered version-gate constants. */
+enum {
+    LEPTRIS_CAP_PARSE = 1ULL << 0,
+    LEPTRIS_CAP_DOM = 1ULL << 1,
+    LEPTRIS_CAP_XPATH = 1ULL << 2,
+    LEPTRIS_CAP_XQUERY = 1ULL << 3,
+    LEPTRIS_CAP_DTD = 1ULL << 4,
+    LEPTRIS_CAP_SAX = 1ULL << 5,
+    LEPTRIS_CAP_HTML = 1ULL << 6,
+    LEPTRIS_CAP_XINCLUDE = 1ULL << 7,
+    LEPTRIS_CAP_DIFF = 1ULL << 8,
+    LEPTRIS_CAP_DIGEST = 1ULL << 9,
+    LEPTRIS_CAP_XSD = 1ULL << 10,
+    LEPTRIS_CAP_XSD_SET = 1ULL << 11,
+    LEPTRIS_CAP_RELAXNG = 1ULL << 12,
+    LEPTRIS_CAP_SCHEMATRON = 1ULL << 13,
+    LEPTRIS_CAP_XSLT = 1ULL << 14,
+    LEPTRIS_CAP_DESCRIPTOR = 1ULL << 15,
+    LEPTRIS_CAP_NS_SNAPSHOT_BULK = 1ULL << 16,
+    LEPTRIS_CAP_CHILDREN_BATCH = 1ULL << 17,
+    LEPTRIS_CAP_NAME_COMPARE = 1ULL << 18,
+    LEPTRIS_CAP_VERSION = 1ULL << 19,
+    /* build-conditional */
+    LEPTRIS_CAP_COLLATION_DUCET = 1ULL << 20,
+    LEPTRIS_CAP_UNICODE_UTF8PROC = 1ULL << 21,
+    LEPTRIS_CAP_ENCODING_ICONV = 1ULL << 22,
+    LEPTRIS_CAP_SIMD_AVX2 = 1ULL << 23,
+    /* LEPTRIS_PARSE_KEEP_ENTITY_REFS yields entity references as
+     * first-class ENTITY_REF nodes (segments split around the five
+     * built-ins and custom named references; leptris#1904). */
+    LEPTRIS_CAP_ENTITY_REF_NODES = 1ULL << 24
+};
+
+/* Descriptor shape version: 1. Bump when the STRUCT changes or a
+ * bit is retired (never expected — bits append). */
+#define LEPTRIS_CAPS_DESCRIPTOR_VERSION 1u
+
+typedef struct leptris_capabilities {
+    unsigned long long caps;
+    unsigned descriptor_version;
+} LeptrisCapabilities;
+
 /* ============================================================================
  * Memory Allocation Function Types
  * ============================================================================ */

data/vendor-src/libleptris/src/include/leptris.h

@@ -4230,6 +4230,12 @@ LEPTRIS_API const char* leptris_version(void);
  */
 LEPTRIS_API void leptris_version_components(int* major, int* minor, int* patch);
 
+/* leptris#1905: the capability descriptor — what this installed
+ * engine serves, as a versioned bitset (LEPTRIS_CAP_*). Query once
+ * at attach; new capabilities append bits (the descriptor version
+ * gates shape changes). Thread-safe: pure, allocation-free. */
+LEPTRIS_API LeptrisCapabilities leptris_capabilities(void);
+
 /**
  * Release this thread's libleptris caches
  *

data/vendor-src/libleptris/src/leptris/common/simd_text.c

@@ -49,6 +49,54 @@ static ptrdiff_t find3_scalar(const char* s, size_t len,
     return -1;
 }
 
+/* Scalar UTF-8 DFA — the always-available fallback (and the oracle
+ * the SIMD cores are pinned against). Moved from the parser's
+ * dp_utf8_valid (#1842 A5). */
+int leptris_utf8_valid_scalar(const char* s_, size_t len) {
+    const unsigned char* p = (const unsigned char*)s_;
+    size_t i = 0;
+    /* ASCII fast path: 8 bytes per iteration. The DFA below runs
+     * only when a high byte is present; ordinary documents pay one
+     * memory-bandwidth pass. */
+    while (i + 8 <= len) {
+        uint64_t w;
+        memcpy(&w, p + i, 8);
+        if (w & UINT64_C(0x8080808080808080)) break;
+        i += 8;
+    }
+    while (i < len && p[i] < 0x80) i++;
+    while (i < len) {
+        unsigned c = p[i];
+        if (c < 0x80) {
+            i++;
+            continue;
+        }
+        size_t n;
+        unsigned lo = 0x80;
+        unsigned hi = 0xBF;
+        if (c >= 0xC2 && c <= 0xDF) {
+            n = 1;
+        } else if (c >= 0xE0 && c <= 0xEF) {
+            n = 2;
+            if (c == 0xE0) lo = 0xA0;
+            else if (c == 0xED) hi = 0x9F;
+        } else if (c >= 0xF0 && c <= 0xF4) {
+            n = 3;
+            if (c == 0xF0) lo = 0x90;
+            else if (c == 0xF4) hi = 0x8F;
+        } else {
+            return 0;
+        }
+        if (len - i <= n) return 0;
+        if (p[i + 1] < lo || p[i + 1] > hi) return 0;
+        for (size_t k = 2; k <= n; k++) {
+            if (p[i + k] < 0x80 || p[i + k] > 0xBF) return 0;
+        }
+        i += n + 1;
+    }
+    return 1;
+}
+
 /* ---- Dispatch ---------------------------------------------------------- */
 
 /* Resolved pointer — initialized on first use (thread-safe enough:
@@ -63,6 +111,7 @@ static void (*g_count3_high_fn)(const char*, size_t, char, char, char,
                                 int*) = NULL;
 static void (*g_copy3_fn)(char*, const char*, size_t, char, char, char,
                           size_t*, size_t*, size_t*) = NULL;
+static int (*g_utf8_fn)(const char*, size_t) = NULL;
 static ptrdiff_t (*g_scan_run_fn)(const char*, size_t, char, char, char,
                                   int*, int*) = NULL;
 
@@ -105,6 +154,9 @@ static size_t count_scalar(const char* s, size_t len, char c) {
  * dispatch_init()" pattern raced when two threads hit the same
  * entry point on first call. */
 static void dispatch_init(void);
+int leptris_utf8_valid_scalar(const char* s, size_t len);
+int leptris_utf8_valid_avx2(const char* s, size_t len);
+int leptris_utf8_valid_neon(const char* s, size_t len);
 static void count3_high_scalar(const char* s, size_t len,
                                char c0, char c1, char c2,
                                size_t* n0, size_t* n1, size_t* n2,
@@ -132,6 +184,7 @@ static void dispatch_init(void) {
         g_count_fn = leptris_text_count_char_avx2;
         g_count3_high_fn = count3_high_scalar;
         g_scan_run_fn = leptris_text_scan_run_avx2;
+        g_utf8_fn = leptris_utf8_valid_avx2;
         return;
     }
 #endif
@@ -144,6 +197,7 @@ static void dispatch_init(void) {
         g_count3_high_fn = leptris_text_count3_high_neon;
         g_copy3_fn = leptris_copy_count3_neon;
         g_scan_run_fn = leptris_text_scan_run_neon;
+        g_utf8_fn = leptris_utf8_valid_neon;
         return;
     }
 #endif
@@ -153,6 +207,7 @@ static void dispatch_init(void) {
     g_count_fn = count_scalar;
     g_count3_high_fn = count3_high_scalar;
     g_scan_run_fn = scan_run_scalar;
+    g_utf8_fn = leptris_utf8_valid_scalar;
 }
 
 /* Scalar kernel: one byte loop counts all three chars and detects
@@ -239,6 +294,12 @@ void leptris_copy_count3(char* dst, const char* src, size_t len,
     leptris_text_count3(src, len, c0, c1, c2, n0, n1, n2);
 }
 
+int leptris_utf8_valid(const char* s, size_t len) {
+    if (!g_utf8_fn) dispatch_init();
+    if (!g_utf8_fn) g_utf8_fn = leptris_utf8_valid_scalar;
+    return g_utf8_fn(s, len);
+}
+
 ptrdiff_t leptris_text_scan_run(const char* s, size_t len, char term,
                                 char f1, char f2,
                                 int* saw_f1, int* saw_f2) {

data/vendor-src/libleptris/src/leptris/common/simd_text.h

@@ -57,6 +57,19 @@ void leptris_text_count3_high(const char* s, size_t len,
                               size_t* n0, size_t* n1, size_t* n2,
                               int* saw_high);
 
+/* Returns 1 when [s, s+len) is valid UTF-8, 0 otherwise. SIMD
+ * boundary-vector validation (the Lemire/Steagall construction) with
+ * the scalar DFA as the fallback; identical verdicts on every input
+ * (pinned by the encoding suite's adversarial matrix + a randomized
+ * cross-check). #1842 A5: the parser's UTF-8 gate ran a per-byte DFA
+ * over the whole document — the hottest line on UTF-8-dense
+ * fixtures (line-level profile, 2026-10-30). */
+int leptris_utf8_valid(const char* s, size_t len);
+
+/* The scalar DFA fallback (and the SIMD verdicts' oracle) — also
+ * the shared tail anchor for the ISA cores. */
+int leptris_utf8_valid_scalar(const char* s, size_t len);
+
 /* Copy [src, src+len) to dst while counting c0/c1/c2 occurrences —
  * one fused memory pass (falls back to memcpy + count3 where no
  * SIMD path is compiled). The parser's owns-copy path gets its