yeptris 0.6.19.1 → 0.6.20.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74a3c58be8335264316d965e0dd3e2e3a63e8861ee05992a957356181530197f
-  data.tar.gz: a33173c381a08cc0f752d2919d2c8abe3f7c7f1660f221a4093afb9f191c2d47
+  metadata.gz: 1d91dc30d6dd414d0b22ce84b0e573850efa6a86e7d6fcef4425e2d4afc2fca2
+  data.tar.gz: 9efe2971ff5b4159b077fba5330a12c0c2ac91bb6849c36c86a83ddeba243a86
 SHA512:
-  metadata.gz: 7564355e04a377764e2116bbac676b6a8f3a8cd4aea0bd869f753386bf9afa29da645181ca29b61ff783beeea3cea341cc999a9d4cc4fdf72c22912c4b3b9721
-  data.tar.gz: e8a2eb033cf888ec75e6657f3ef186a94ded424aeef5e88e63bfd6b7f2c353514931fe2775f19e158722d9c94d10d713bef625a8a7a02fe6af521c1523cc8fe7
+  metadata.gz: b40b768c6103cc5715059d7f93350c8c2f9298f86297e2133cf339ec269529237a964bcab0ef5a6ea998a609b5370ac640ba6d195b741bd9382037e2a3114a83
+  data.tar.gz: ecfe46a3b63e70b38c11541f6926b7e59a622e8450401b0ff271a362d854317f4bc40647b9ffa2bda3233db3b1e3f09470385f8768f27b794b9655a387123ed4

data/ext/build_windows_native.rb

@@ -1,16 +1,25 @@
 # frozen_string_literal: true
 
-# Builds the native materializer DLL for the CURRENT Ruby and
-# installs it under its minor-versioned name
-# (lib/yeptris/native-<major.minor>.so).
+# Builds the native materializer for the CURRENT Ruby and installs it
+# under the per-minor DIRECTORY layout
+# (lib/yeptris/<major.minor>/native.<so|bundle|dll>).
 #
 # The leptris-ruby Windows lesson (#207/#227): a PE DLL cannot
 # resolve Ruby imports lazily — it must bind the build Ruby's
-# x64-ucrt-rubyNNN.dll. One DLL per supported minor therefore ships
-# in the Windows platform gems, and the loader in lib/yeptris.rb
-# picks by RUBY_VERSION at require. Ruby 3.3 has no arm64 build:
-# that cell ships no artifact and falls back loudly to the FFI
-# ladder.
+# x64-ucrt-rubyNNN.dll. One bundle per supported minor therefore ships
+# in the platform gems, and the loader in lib/yeptris.rb picks by
+# RUBY_VERSION at require. Ruby 3.3 has no arm64 build: that cell
+# ships no artifact and falls back loudly to the FFI ladder.
+#
+# The MINOR goes in the directory and the file name stays "native":
+# ruby derives the init symbol from the feature basename, so the old
+# native-<minor>.so name made require ask for the untypeable
+# "Init_native-3" and no shipped bundle could ever load (#157).
+#
+# Unix bundles link the same shape build-platform-gem.sh uses — no
+# libruby DT_NEEDED (its absolute build-runner path dangles on user
+# machines; Ruby API symbols resolve from the host process), and
+# darwin links with -undefined dynamic_lookup.
 #
 # Standalone by design: the release workflow runs this under each
 # Ruby minor (3.3/3.4/4.0) with no bundler context.
@@ -21,21 +30,31 @@ require "fileutils"
 root = File.expand_path("..", __dir__)
 ext_dir = File.join(root, "ext", "yeptris_native")
 minor = RUBY_VERSION[/\A\d+\.\d+/]
+windows = RbConfig::CONFIG["host_os"] =~ /mingw|mswin/
 
 Dir.chdir(ext_dir) do
   system(RbConfig.ruby, "extconf.rb") or abort "extconf failed under #{RUBY_VERSION}"
   # mkmf's link binds the CURRENT Ruby's runtime DLL — exactly
-  # what the versioned naming is for.
-  success = system("make")
+  # what the per-minor scheme is for.
+  if windows
+    success = system("make")
+  else
+    darwin = RbConfig::CONFIG["host_os"] =~ /darwin/
+    dld = darwin ? "-dynamic -bundle -undefined dynamic_lookup" : ""
+    success = system("make", "LIBRUBYARG_SHARED=", "LIBRUBYARG_STATIC=",
+                     "DLDFLAGS=#{dld}")
+  end
   abort "make failed under #{RUBY_VERSION}" unless success
-  so = Dir.glob("native.{so,dll}").first
+  so = Dir.glob("native.{so,bundle,dll}").first
   abort "native bundle not produced under #{RUBY_VERSION}" unless so
-  dest = File.join(root, "lib", "yeptris", "native-#{minor}.so")
+  dest_dir = File.join(root, "lib", "yeptris", minor)
+  dest = File.join(dest_dir, File.basename(so))
   # The artifact must reference only this minor's Ruby DLL.
   imported = `strings #{so} 2>/dev/null`[/[a-z0-9-]*ruby\d{3,}\.dll/i]
   if imported && !imported.include?("ruby#{minor.delete('.')}")
     abort "#{so} imports #{imported} but was built under #{RUBY_VERSION} — refuse to mis-name it"
   end
+  FileUtils.mkdir_p(dest_dir)
   FileUtils.cp(so, dest)
   puts "Installed native materializer for Ruby #{minor} -> #{dest} (#{imported || 'no ruby dll string found'})"
 end

data/lib/yeptris/psych.rb

@@ -619,7 +619,7 @@ end
 # rebind ::Psych. It does not anymore, and re-binding it here would
 # regress #69 (any stdlib psych loaded afterwards would explode with
 # a superclass mismatch), so the old path warns instead of acting.
-if defined?(::Psych) && !::Psych.equal?(Yeptris::Psych)
+if defined?(::Psych) && !::Psych.equal?(Yeptris::Psych) && Yeptris.debug?
   warn "yeptris: \"yeptris/psych\" defines the namespace only — it no " \
        "longer rebinds ::Psych. Require \"yeptris/psych/drop_in\" for the " \
        "drop-in rebind, or call Yeptris::Psych explicitly."

data/lib/yeptris.rb

@@ -3,7 +3,17 @@
 module Yeptris
   # The gem's version lives in the parent namespace's file — the last
   # internal require (yeptris/version) retired with it.
-  VERSION = "0.6.19.1".freeze
+  VERSION = "0.6.20.2".freeze
+
+  # Informational notices are OPT-IN (yeptris-ruby#217): consumers
+  # embed yeptris inside CLIs that assert clean stderr, and a
+  # successful fallback is not a warning. YEPTRIS_DEBUG=1 turns the
+  # notices on.
+  def self.debug?
+    return @debug unless @debug.nil?
+    v = ENV["YEPTRIS_DEBUG"]
+    @debug = !v.nil? && !v.empty? && v != "0"
+  end
   # The error hierarchy lives in THIS file (the parent namespace's
   # own file): nested constants do not trigger a parent-constant
   # autoload, and the law forbids internal requires — defining the
@@ -91,23 +101,29 @@ end
 # Optional C-API materializer (TODO.restructure/22): fused visit →
 # Ruby objects via the Ruby C API. Feature-detected — LoadError leaves
 # the FFI ladder (Marshal → columns → records) as the sole path.
-# Windows names the artifact per Ruby minor (the leptris-ruby #207/
-# #227 lesson): a PE DLL must bind its build Ruby's runtime, so one
-# DLL per supported minor ships and RUBY_VERSION selects. A missing
-# cell (Ruby 3.3 arm64 has no build) falls back LOUDLY to the ladder.
+# One bundle per Ruby minor (the leptris-ruby #207/#227 lesson: a PE
+# DLL must bind its build Ruby's runtime; a unix bundle is also
+# minor-locked by the internal API it compiles against). The MINOR
+# lives in the DIRECTORY (lib/yeptris/<minor>/native.<ext>) and the
+# FILE name stays unversioned: ruby derives the init symbol from the
+# feature basename, so native-3.4.so asked for the untypeable
+# "Init_native-3" and the versioned-file scheme could never load
+# (#157 — every shipped bundle was dead code). Dev checkouts keep
+# the plain native.<ext> name (mkmf layout, built for the running
+# Ruby). A missing cell falls back LOUDLY to the ladder.
 begin
-  # per-minor FIRST everywhere: a single version-agnostic native.so
-  # loads under ANY Ruby (symbols resolve from the host process), and
-  # one built for a different minor is an ABI gamble — Ruby 3.0 ran
-  # a 3.3-built ext and the JSON parse failed. The plain name stays
-  # as the dev-checkout fallback (built for the running Ruby).
-  require "yeptris/native-#{RUBY_VERSION[/\A\d+\.\d+/]}"
+  # per-minor FIRST: a single version-agnostic native.so loads under
+  # ANY Ruby and is an ABI gamble across minors — Ruby 3.0 ran a
+  # 3.3-built ext and the JSON parse failed.
+  require "yeptris/#{RUBY_VERSION[/\A\d+\.\d+/]}/native"
 rescue LoadError
   begin
     require "yeptris/native"
   rescue LoadError => e
-    warn "yeptris: no precompiled native materializer for Ruby " \
-         "#{RUBY_VERSION[/\A\d+\.\d+/]} on this platform " \
-         "(#{e.message}); the FFI ladder carries the load"
+    if Yeptris.debug? # the require chain is top-level (self = main)
+      warn "yeptris: no precompiled native materializer for Ruby " \
+           "#{RUBY_VERSION[/\A\d+\.\d+/]} on this platform " \
+           "(#{e.message}); the FFI ladder carries the load"
+    end
   end
 end

data/vendor/libyeptris/CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(yeptris
-    VERSION 0.6.19
+    VERSION 0.6.20
     DESCRIPTION "Ultra-fast YAML 1.2 parser, emitter and streamer in C"
     LANGUAGES C CXX
 )

data/vendor/libyeptris/src/CMakeLists.txt

@@ -24,6 +24,7 @@ set(YEPTRIS_SOURCES
     yeptris/parse/scalars.c
     yeptris/parse/engine.c
     yeptris/dom/dom.c
+    yeptris/dom/ytape.c
     yeptris/dom/hpool.c
     yeptris/dom/mapindex.c
     yeptris/dom/mutate.c

data/vendor/libyeptris/src/yeptris/build.c

@@ -31,6 +31,7 @@ YEPTRIS_API YeptrisDocument yeptris_document_new(void) {
         yep_dom_destroy(dom);
         return NULL;
     }
+    yep_mutex_init(&doc->lazy_mu);
     memset(doc, 0, sizeof(*doc));
     doc->dom = dom;
     doc->sys = sys;

data/vendor/libyeptris/src/yeptris/cbor/decode.c

@@ -860,6 +860,7 @@ static YeptrisDocument cbor_wrap(yep_dom* dom, const void* buf, const yep_alloca
         yep_dom_destroy(dom);
         return NULL;
     }
+    yep_mutex_init(&d->lazy_mu);
     d->dom = dom;
     d->sys = sys;
     d->schema = YEPTRIS_SCHEMA_12_CORE;
@@ -868,6 +869,7 @@ static YeptrisDocument cbor_wrap(yep_dom* dom, const void* buf, const yep_alloca
     d->input = (const char*)buf;
     d->finish_pool = NULL;
     d->lazy_tape = NULL; /* field-by-field ctor: no garbage for free */
+    d->lazy_kind = 0;
     return (YeptrisDocument)d;
 }
 
@@ -966,6 +968,7 @@ YEPTRIS_API YeptrisDocument yeptris_cbor_decode(const void* buf, size_t len, uin
         }
         return NULL;
     }
+    yep_mutex_init(&doc->lazy_mu);
     doc->dom = dom;
     doc->sys = sys;
     doc->schema = YEPTRIS_SCHEMA_12_CORE; /* CBOR's model is the core schema */

data/vendor/libyeptris/src/yeptris/common/simd_text.h

@@ -99,6 +99,15 @@ typedef struct yep_chunk_masks {
     uint32_t valid;    /* 1-bits for the bytes that exist (tails) */
 } yep_chunk_masks;
 
+/* one 64-byte block's classification (json_stage1_masks) */
+struct yep_s1_block {
+    uint64_t q;      /* raw quote bytes */
+    uint64_t bs;     /* backslash bytes */
+    uint64_t op;     /* []{}:, */
+    uint64_t c0;     /* bytes < 0x20 (ws included) */
+    uint64_t tokens; /* the resolver's verdict: ops + string-open quotes */
+};
+
 /* The kernel table. One struct = one dispatch point (OCP: a new ISA is a
  * new TU exporting a new table; nothing else changes). */
 typedef struct yep_text_kernels {
@@ -175,10 +184,25 @@ typedef struct yep_text_kernels {
      * hard error is an unterminated string (odd final parity) ->
      * return 0. idx must hold len+2 entries. Bit-identical across
      * ISAs (the differential suite pins it); scan/json.c owns the
-     * scalar reference and the grammar contract. */
-    int (*json_stage1)(const char* p, size_t len, uint32_t* idx, size_t* nidx);
+     * grammar contract. flags (never NULL) accumulates YEP_S1_* bits:
+     * the doc-level facts stage 2 needs to skip content scanning. */
+    int (*json_stage1)(const char* p, size_t len, uint32_t* idx, size_t* nidx, unsigned* flags);
+
+    /* The structural indexer's classification half, without the
+     * emission: per 64-byte block, the four u64 masks the resolver
+     * consumes (QUOTE/BS/OP/C0; ws is dead since the token contract
+     * dropped scalar-run starts). blocks receives len/64+1 records;
+     * *nblocks returns how many were written. Same reject/flag
+     * contract as json_stage1. The single-pass tape walk consumes
+     * this — no idx array, no emission loop. */
+    int (*json_stage1_masks)(const char* p, size_t len, struct yep_s1_block* blocks,
+                             size_t* nblocks, unsigned* flags);
 } yep_text_kernels;
 
+/* json_stage1 flag bits (accumulated across blocks; cleared by caller) */
+#define YEP_S1_C0_IN_STRING 1u /* raw non-ws C0 inside a string: reject */
+#define YEP_S1_HAS_ESCAPE 2u   /* a backslash byte exists: validate spans */
+
 /* The structural indexer's mask resolver — the SSOT identity set for
  * every json_stage1 implementation (scalar in scan/json.c, the ISA
  * twins in the SIMD TUs). Given one 64-byte block's QUOTE/BS/OP/WS
@@ -195,15 +219,13 @@ typedef struct yep_text_kernels {
  *   tokens     = (op | starts) & ~string_tail
  *
  * Returns the block's contribution; carries update in place. */
-static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t op, uint64_t ws,
-                                             uint64_t valid, uint64_t* prev_in_string,
-                                             uint64_t* esc_carry, uint64_t* follows_carry,
-                                             size_t off, uint32_t* idx, size_t n) {
+static inline uint64_t yep_json_stage1_tokens(uint64_t q, uint64_t bs, uint64_t op, uint64_t c0,
+                                              uint64_t valid, uint64_t* prev_in_string,
+                                              uint64_t* esc_carry, unsigned* flags) {
     uint64_t potential = bs & ~*esc_carry;
     uint64_t s = (((potential << 1) | 0xAAAAAAAAAAAAAAAAull) - potential) ^ 0xAAAAAAAAAAAAAAAAull;
-    uint64_t esc_bs = s & bs;
     uint64_t escaped = s ^ (bs | *esc_carry);
-    *esc_carry = esc_bs >> 63;
+    *esc_carry = (s & bs) >> 63;
     uint64_t quote_live = q & ~escaped;
     uint64_t ps = quote_live; /* sequential prefix-xor — a flat
                                * one-expression xor of shifts misses
@@ -217,15 +239,27 @@ static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t o
     uint64_t in_string = ps ^ *prev_in_string;
     *prev_in_string = (uint64_t)((int64_t)in_string >> 63);
     uint64_t string_tail = (in_string ^ quote_live) & valid;
-    uint64_t scalar_nq = (~(op | ws) & ~q) & valid; /* non-quote scalars */
-    uint64_t follows = ((scalar_nq & ~(1ull << 63)) << 1) | *follows_carry;
-    *follows_carry = scalar_nq >> 63;
-    uint64_t tokens = ((op | (~(op | ws) & ~follows)) & ~string_tail) & valid;
+    if (bs != 0) {
+        *flags |= YEP_S1_HAS_ESCAPE;
+    }
+    if ((c0 & in_string & valid) != 0) {
+        *flags |= YEP_S1_C0_IN_STRING;
+    }
+    return (op | (quote_live & ~string_tail)) & ~string_tail & valid;
+}
+
+/* The full resolver: token mask derivation + the idx emission loop
+ * (the json_stage1 implementations). */
+static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t op, uint64_t ws,
+                                             uint64_t c0, uint64_t valid, uint64_t* prev_in_string,
+                                             uint64_t* esc_carry, size_t off, uint32_t* idx,
+                                             size_t n, unsigned* flags) {
+    uint64_t tokens =
+        yep_json_stage1_tokens(q, bs, op, c0, valid, prev_in_string, esc_carry, flags);
     while (tokens != 0) {
         idx[n++] = (uint32_t)(off + (size_t)yep_ctz64(tokens));
         tokens &= tokens - 1;
     }
-    (void)escaped;
     return n;
 }
 

data/vendor/libyeptris/src/yeptris/common/simd_text_avx2.c

@@ -455,12 +455,13 @@ yep_chunk_masks yep_text_json_chunk_avx2(const char* p, size_t n) {
  * (two 32-byte lane passes each, movemask-native), then the shared
  * mask resolver (simd_text.h). The final partial block builds its
  * masks byte-wise — no loads past len. */
-int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t* nidx) {
+int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                              unsigned* flags) {
     size_t n = 0;
-    uint64_t prev_in_string = 0, esc_carry = 0, follows_carry = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
     size_t off = 0;
     for (; off + 64 <= len; off += 64) {
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (unsigned half = 0; half < 2; half++) {
             const __m256i v =
                 _mm256_loadu_si256((const __m256i*)(const void*)(p + off + 32 * half));
@@ -482,17 +483,21 @@ int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t*
                                 _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\n')),
                                                 _mm256_cmpeq_epi8(v, _mm256_set1_epi8('\r'))));
             uint64_t fw = (uint64_t)(uint32_t)_mm256_movemask_epi8(w);
+            /* unsigned min(v, 0x20) == v  <=>  v < 0x20 */
+            uint64_t fc = (uint64_t)(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+                _mm256_max_epu8(v, _mm256_set1_epi8(0x1F)), _mm256_set1_epi8(0x1F)));
             q |= fq << (32 * half);
             bs |= fb << (32 * half);
             op |= fo << (32 * half);
             ws |= fw << (32 * half);
+            c0 |= fc << (32 * half);
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, ~0ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, ~0ull, &prev_in_string, &esc_carry, off, idx,
+                                    n, flags);
     }
     if (off < len) {
         size_t cn = len - off;
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (size_t k = 0; k < cn; k++) {
             unsigned char c = (unsigned char)p[off + k];
             uint64_t bit = 1ull << k;
@@ -505,14 +510,86 @@ int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t*
             } else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
                 ws |= bit;
             }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, (1ull << cn) - 1ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, (1ull << cn) - 1ull, &prev_in_string,
+                                    &esc_carry, off, idx, n, flags);
     }
     *nidx = n;
     return prev_in_string ? 0 : 1;
 }
 
+/* The classification half of the structural indexer (AVX2 twin of the
+ * NEON masks slot): two 32-byte lane passes per 64-byte block. */
+int yep_text_json_stage1_masks_avx2(const char* p, size_t len, struct yep_s1_block* blocks,
+                                    size_t* nblocks, unsigned* flags) {
+    size_t nb = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
+    size_t off = 0;
+    for (; off + 64 <= len; off += 64) {
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (unsigned half = 0; half < 2; half++) {
+            const __m256i v =
+                _mm256_loadu_si256((const __m256i*)(const void*)(p + off + 32 * half));
+            uint64_t fq = (uint64_t)(uint32_t)_mm256_movemask_epi8(
+                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"')));
+            uint64_t fb = (uint64_t)(uint32_t)_mm256_movemask_epi8(
+                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\')));
+            __m256i o = _mm256_or_si256(
+                _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('{')),
+                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('}'))),
+                _mm256_or_si256(_mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('[')),
+                                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8(']'))),
+                                _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8(',')),
+                                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8(':')))));
+            uint64_t fo = (uint64_t)(uint32_t)_mm256_movemask_epi8(o);
+            uint64_t fc = (uint64_t)(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+                _mm256_max_epu8(v, _mm256_set1_epi8(0x1F)), _mm256_set1_epi8(0x1F)));
+            q |= fq << (32 * half);
+            bs |= fb << (32 * half);
+            op |= fo << (32 * half);
+            c0 |= fc << (32 * half);
+        }
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, ~0ull, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    if (off < len) {
+        size_t cn = len - off;
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (size_t k = 0; k < cn; k++) {
+            unsigned char c = (unsigned char)p[off + k];
+            uint64_t bit = 1ull << k;
+            if (c == '"') {
+                q |= bit;
+            } else if (c == '\\') {
+                bs |= bit;
+            } else if (c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == ':') {
+                op |= bit;
+            }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
+        }
+        uint64_t valid = (1ull << cn) - 1ull;
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, valid, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    *nblocks = nb;
+    return prev_in_string ? 0 : 1;
+}
+
 const yep_text_kernels yep_text_kernels_avx2 = {
     yep_avx2_contains,
     yep_avx2_find,
@@ -529,6 +606,7 @@ const yep_text_kernels yep_text_kernels_avx2 = {
     yep_avx2_line_facts,
     yep_text_json_chunk_avx2,
     yep_text_json_stage1_avx2,
+    yep_text_json_stage1_masks_avx2,
 };
 
 #endif /* YEP_ARCH_X86 */

data/vendor/libyeptris/src/yeptris/common/simd_text_neon.c

@@ -588,9 +588,10 @@ yep_chunk_masks yep_text_json_chunk_neon(const char* p, size_t n) {
  * (four 16-byte halves each), then the shared mask resolver (the
  * simdjson identity set, simd_text.h). The final partial block
  * builds its masks byte-wise — no loads past len. */
-int yep_text_json_stage1_neon(const char* p, size_t len, uint32_t* idx, size_t* nidx) {
+int yep_text_json_stage1_neon(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                              unsigned* flags) {
     size_t n = 0;
-    uint64_t prev_in_string = 0, esc_carry = 0, follows_carry = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
     size_t off = 0;
     const uint8x16_t dq = vdupq_n_u8('"'), dbs = vdupq_n_u8('\\');
     const uint8x16_t d1 = vdupq_n_u8('{'), d2 = vdupq_n_u8('}');
@@ -598,8 +599,9 @@ int yep_text_json_stage1_neon(const char* p, size_t len, uint32_t* idx, size_t*
     const uint8x16_t d5 = vdupq_n_u8(','), d6 = vdupq_n_u8(':');
     const uint8x16_t wsp = vdupq_n_u8(' '), wtab = vdupq_n_u8('\t');
     const uint8x16_t wnl = vdupq_n_u8('\n'), wcr = vdupq_n_u8('\r');
+    const uint8x16_t c0t = vdupq_n_u8(0x20);
     for (; off + 64 <= len; off += 64) {
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (unsigned half = 0; half < 4; half++) {
             uint8x16_t v = vld1q_u8((const uint8_t*)(const void*)(p + off + 16 * half));
             uint64_t fq = yep_neon_bits(vceqq_u8(v, dq));
@@ -610,17 +612,19 @@ int yep_text_json_stage1_neon(const char* p, size_t len, uint32_t* idx, size_t*
                                        vorrq_u8(vceqq_u8(v, d5), vceqq_u8(v, d6))));
             uint64_t fw = yep_neon_bits(vorrq_u8(vorrq_u8(vceqq_u8(v, wsp), vceqq_u8(v, wtab)),
                                                  vorrq_u8(vceqq_u8(v, wnl), vceqq_u8(v, wcr))));
+            uint64_t fc = yep_neon_bits(vcltq_u8(v, c0t));
             q |= fq << (16 * half);
             bs |= fb << (16 * half);
             op |= fo << (16 * half);
             ws |= fw << (16 * half);
+            c0 |= fc << (16 * half);
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, ~0ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, ~0ull, &prev_in_string, &esc_carry, off, idx,
+                                    n, flags);
     }
     if (off < len) { /* the tail: byte-wise masks, no loads past len */
         size_t cn = len - off;
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (size_t k = 0; k < cn; k++) {
             unsigned char c = (unsigned char)p[off + k];
             uint64_t bit = 1ull << k;
@@ -633,14 +637,83 @@ int yep_text_json_stage1_neon(const char* p, size_t len, uint32_t* idx, size_t*
             } else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
                 ws |= bit;
             }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, (1ull << cn) - 1ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, (1ull << cn) - 1ull, &prev_in_string,
+                                    &esc_carry, off, idx, n, flags);
     }
     *nidx = n;
     return prev_in_string ? 0 : 1;
 }
 
+/* The classification half of the structural indexer: the same
+ * 64-byte/4-half mask build, no emission, blocks out instead of idx. */
+int yep_text_json_stage1_masks_neon(const char* p, size_t len, struct yep_s1_block* blocks,
+                                    size_t* nblocks, unsigned* flags) {
+    size_t nb = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
+    size_t off = 0;
+    const uint8x16_t dq = vdupq_n_u8('"'), dbs = vdupq_n_u8('\\');
+    const uint8x16_t d1 = vdupq_n_u8('{'), d2 = vdupq_n_u8('}');
+    const uint8x16_t d3 = vdupq_n_u8('['), d4 = vdupq_n_u8(']');
+    const uint8x16_t d5 = vdupq_n_u8(','), d6 = vdupq_n_u8(':');
+    const uint8x16_t c0t = vdupq_n_u8(0x20);
+    for (; off + 64 <= len; off += 64) {
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (unsigned half = 0; half < 4; half++) {
+            uint8x16_t v = vld1q_u8((const uint8_t*)(const void*)(p + off + 16 * half));
+            uint64_t fq = yep_neon_bits(vceqq_u8(v, dq));
+            uint64_t fb = yep_neon_bits(vceqq_u8(v, dbs));
+            uint64_t fo =
+                yep_neon_bits(vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(v, d1), vceqq_u8(v, d2)),
+                                                vorrq_u8(vceqq_u8(v, d3), vceqq_u8(v, d4))),
+                                       vorrq_u8(vceqq_u8(v, d5), vceqq_u8(v, d6))));
+            uint64_t fc = yep_neon_bits(vcltq_u8(v, c0t));
+            q |= fq << (16 * half);
+            bs |= fb << (16 * half);
+            op |= fo << (16 * half);
+            c0 |= fc << (16 * half);
+        }
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, ~0ull, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    if (off < len) {
+        size_t cn = len - off;
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (size_t k = 0; k < cn; k++) {
+            unsigned char c = (unsigned char)p[off + k];
+            uint64_t bit = 1ull << k;
+            if (c == '"') {
+                q |= bit;
+            } else if (c == '\\') {
+                bs |= bit;
+            } else if (c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == ':') {
+                op |= bit;
+            }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
+        }
+        uint64_t valid = (1ull << cn) - 1ull;
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, valid, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    *nblocks = nb;
+    return prev_in_string ? 0 : 1;
+}
+
 const yep_text_kernels yep_text_kernels_neon = {
     yep_neon_contains,
     yep_neon_find,
@@ -657,6 +730,7 @@ const yep_text_kernels yep_text_kernels_neon = {
     yep_neon_line_facts,
     yep_text_json_chunk_neon,
     yep_text_json_stage1_neon,
+    yep_text_json_stage1_masks_neon,
 };
 
 #endif /* YEP_ARCH_AARCH64 */

data/vendor/libyeptris/src/yeptris/common/simd_text_scalar.c

@@ -287,12 +287,16 @@ yep_chunk_masks yep_text_json_chunk_scalar(const char* p, size_t n) {
 
 /* the stage-1 scalar reference lives in scan/json.c (the grammar
  * home); the table carries it so dispatch stays one point */
-extern int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx);
+extern int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                                  unsigned* flags);
+extern int yep_json_stage1_masks_scalar(const char* p, size_t len, struct yep_s1_block* blocks,
+                                        size_t* nblocks, unsigned* flags);
 
 const yep_text_kernels yep_text_kernels_scalar = {
-    yep_text_contains_scalar,   yep_text_find_scalar,         yep_text_find3_scalar,
-    yep_text_count_char_scalar, yep_text_count3_scalar,       yep_text_copy_count3_scalar,
-    yep_text_find_not_scalar,   yep_text_stopset_find_scalar, yep_text_quote_scan_scalar,
-    yep_text_scan_stats_scalar, yep_text_qbc_find_scalar,     yep_text_gate_scan_scalar,
-    yep_text_line_facts_scalar, yep_text_json_chunk_scalar,   yep_json_stage1_scalar,
+    yep_text_contains_scalar,     yep_text_find_scalar,         yep_text_find3_scalar,
+    yep_text_count_char_scalar,   yep_text_count3_scalar,       yep_text_copy_count3_scalar,
+    yep_text_find_not_scalar,     yep_text_stopset_find_scalar, yep_text_quote_scan_scalar,
+    yep_text_scan_stats_scalar,   yep_text_qbc_find_scalar,     yep_text_gate_scan_scalar,
+    yep_text_line_facts_scalar,   yep_text_json_chunk_scalar,   yep_json_stage1_scalar,
+    yep_json_stage1_masks_scalar,
 };

data/vendor/libyeptris/src/yeptris/doc.h

@@ -7,6 +7,7 @@
 
 #include <stddef.h>
 
+#include "common/mutex.h"
 #include "dom/dom.h"
 #include "memory/allocator.h"
 
@@ -18,15 +19,24 @@ typedef struct yeptris_document {
                                 * conditioned on it (TODO.restructure/32) */
     unsigned char* transcoded; /* owned when non-NULL */
     size_t transcoded_len;
-    const char* input; /* borrowed input (lifetime documentation) */
-    void* finish_pool; /* engine finish pool: resolved tags, folded and
-                          escaped scalars outlive the engine through the
-                          document */
-    void* lazy_tape;   /* #342 slice 2: the parsed tape when the tree is
-                        * deferred (gate-clean strict JSON). dom==NULL until
-                        * the first tree access materializes via dom_from_tape;
-                        * freed with the document. void* to avoid a header
-                        * cycle — parse.c casts to yeptris_json_tape* */
+    const char* input;     /* borrowed input (lifetime documentation) */
+    void* finish_pool;     /* engine finish pool: resolved tags, folded and
+                              escaped scalars outlive the engine through the
+                              document */
+    void* lazy_tape;       /* #342 slice 2: the parsed tape when the tree is
+                            * deferred (gate-clean strict JSON; #378: the YAML
+                            * record tape). dom==NULL until the first tree
+                            * access materializes; freed with the document.
+                            * void* to avoid a header cycle — parse.c casts
+                            * per lazy_kind */
+    int lazy_kind;         /* 0 = none/json (yeptris_json_tape), 1 = yaml
+                            * (yep_ytape) — every constructor must set it
+                            * (the field-by-field ctor trap) */
+    yep_mutex_raw lazy_mu; /* materialization lock: the first tree
+                            * access builds; concurrent first accesses
+                            * wait (read-only sharing stays lock-free
+                            * once dom is set — every constructor must
+                            * yep_mutex_init it, the ctor trap) */
 } yeptris_document;
 
 /* Node handle: a (document, node-id) pair so nodes stay usable even if