yeptris 0.6.18.2 → 0.6.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d2c20e34ee04e8ead36d9dd1d0416a69e3171bdd9d567e724a9781308aa5287
-  data.tar.gz: 9e2c440e6ab072af22c568dd7de4262b331d3bcd69d61dad25dae64854d6a925
+  metadata.gz: e22b16a8d0158fb8161f390d54b7dd8de6612fbfcc881741a7896f0a891b6188
+  data.tar.gz: 9fab611eff46efb8a82f434a6daf89fafbd400c3493ee80a36a01fbcba7a8983
 SHA512:
-  metadata.gz: b3c9fd2d90bd3490fa8070c47088a10288dd5933098926a3a703ad9e78fd44a06bc2fd38c3f5cd4b373ca056faa6db6f0d4004a06d415be964470b6989423acd
-  data.tar.gz: 89da80d2595be33241cc7b694ae69a262cf58d8d8dbb955d7330357572783d6a6f07cf9ebb56553fa9313e977361f6460547cc6f4680d63642f30140d73e27ae
+  metadata.gz: 66e8f18cd12134048d4a89d95f4d3c93faeb2f0700e0e31a9ce34b12269feba643c4e896fe2d5b3bc0d36958439e97901b9b2e376fc9afc1c62e455e70af2c66
+  data.tar.gz: c983b4b04736ae785e8c3b6c4380968986eaf105326b2d3e10c155f9d86fe0522cd76cb940649685ff11b4d510794828d28e56a737d4e9afda23badceb9262da

data/ext/build_windows_native.rb

@@ -1,16 +1,25 @@
 # frozen_string_literal: true
 
-# Builds the native materializer DLL for the CURRENT Ruby and
-# installs it under its minor-versioned name
-# (lib/yeptris/native-<major.minor>.so).
+# Builds the native materializer for the CURRENT Ruby and installs it
+# under the per-minor DIRECTORY layout
+# (lib/yeptris/<major.minor>/native.<so|bundle|dll>).
 #
 # The leptris-ruby Windows lesson (#207/#227): a PE DLL cannot
 # resolve Ruby imports lazily — it must bind the build Ruby's
-# x64-ucrt-rubyNNN.dll. One DLL per supported minor therefore ships
-# in the Windows platform gems, and the loader in lib/yeptris.rb
-# picks by RUBY_VERSION at require. Ruby 3.3 has no arm64 build:
-# that cell ships no artifact and falls back loudly to the FFI
-# ladder.
+# x64-ucrt-rubyNNN.dll. One bundle per supported minor therefore ships
+# in the platform gems, and the loader in lib/yeptris.rb picks by
+# RUBY_VERSION at require. Ruby 3.3 has no arm64 build: that cell
+# ships no artifact and falls back loudly to the FFI ladder.
+#
+# The MINOR goes in the directory and the file name stays "native":
+# ruby derives the init symbol from the feature basename, so the old
+# native-<minor>.so name made require ask for the untypeable
+# "Init_native-3" and no shipped bundle could ever load (#157).
+#
+# Unix bundles link the same shape build-platform-gem.sh uses — no
+# libruby DT_NEEDED (its absolute build-runner path dangles on user
+# machines; Ruby API symbols resolve from the host process), and
+# darwin links with -undefined dynamic_lookup.
 #
 # Standalone by design: the release workflow runs this under each
 # Ruby minor (3.3/3.4/4.0) with no bundler context.
@@ -21,21 +30,31 @@ require "fileutils"
 root = File.expand_path("..", __dir__)
 ext_dir = File.join(root, "ext", "yeptris_native")
 minor = RUBY_VERSION[/\A\d+\.\d+/]
+windows = RbConfig::CONFIG["host_os"] =~ /mingw|mswin/
 
 Dir.chdir(ext_dir) do
   system(RbConfig.ruby, "extconf.rb") or abort "extconf failed under #{RUBY_VERSION}"
   # mkmf's link binds the CURRENT Ruby's runtime DLL — exactly
-  # what the versioned naming is for.
-  success = system("make")
+  # what the per-minor scheme is for.
+  if windows
+    success = system("make")
+  else
+    darwin = RbConfig::CONFIG["host_os"] =~ /darwin/
+    dld = darwin ? "-dynamic -bundle -undefined dynamic_lookup" : ""
+    success = system("make", "LIBRUBYARG_SHARED=", "LIBRUBYARG_STATIC=",
+                     "DLDFLAGS=#{dld}")
+  end
   abort "make failed under #{RUBY_VERSION}" unless success
-  so = Dir.glob("native.{so,dll}").first
+  so = Dir.glob("native.{so,bundle,dll}").first
   abort "native bundle not produced under #{RUBY_VERSION}" unless so
-  dest = File.join(root, "lib", "yeptris", "native-#{minor}.so")
+  dest_dir = File.join(root, "lib", "yeptris", minor)
+  dest = File.join(dest_dir, File.basename(so))
   # The artifact must reference only this minor's Ruby DLL.
   imported = `strings #{so} 2>/dev/null`[/[a-z0-9-]*ruby\d{3,}\.dll/i]
   if imported && !imported.include?("ruby#{minor.delete('.')}")
     abort "#{so} imports #{imported} but was built under #{RUBY_VERSION} — refuse to mis-name it"
   end
+  FileUtils.mkdir_p(dest_dir)
   FileUtils.cp(so, dest)
   puts "Installed native materializer for Ruby #{minor} -> #{dest} (#{imported || 'no ruby dll string found'})"
 end

data/lib/yeptris/materializer.rb

@@ -12,9 +12,10 @@ module Yeptris
   # materializer (Node#to_ruby) stays for node-based use; this is the
   # Yeptris::YAML fast path.
   class Materializer
-    RECORD_SIZE = 36 # YeptrisEventRecord layout (events.h, ABI-pinned)
-    FIELDS = 12      # unpacked values per record
-    RECORD_UNPACK = "C4V8" # type/style/flags/tag_id, line..tag_len
+    RECORD_SIZE = 44 # YeptrisEventRecord layout (events.h, ABI-pinned;
+                     # #179 added end_line/end_col in the 0.6.19 window)
+    FIELDS = 14      # unpacked values per record
+    RECORD_UNPACK = "C4V10" # type/style/flags/tag_id, line..tag_len
 
     STREAM_START = 1
     STREAM_END = 2
@@ -256,10 +257,10 @@ module Yeptris
           # not the text
           tag_id = flat[i + 3]
           v = Materializer.scan_by_tag(
-            arena[flat[i + 6], flat[i + 7]], tag_id, (flat[i + 2] & EF_IMPLICIT) != 0
+            arena[flat[i + 8], flat[i + 9]], tag_id, (flat[i + 2] & EF_IMPLICIT) != 0
           )
-          l = flat[i + 9]
-          anchors[arena[flat[i + 8], l]] = v if l != 0
+          l = flat[i + 11]
+          anchors[arena[flat[i + 10], l]] = v if l != 0
           # place(): the scalar fast path inlined — the overwhelming
           # majority of events land here
           if stack.empty?
@@ -284,22 +285,22 @@ module Yeptris
           end
         when MAPPING_START
           h = {}
-          l = flat[i + 9]
-          anchors[arena[flat[i + 8], l]] = h if l != 0
+          l = flat[i + 11]
+          anchors[arena[flat[i + 10], l]] = h if l != 0
           merge_target.push(place(docs, stack, pending_key, pending_key_merge, h))
           stack.push(h)
           pending_key.push(nil)
           pending_key_merge.push(nil)
         when SEQUENCE_START
           a = []
-          l = flat[i + 9]
-          anchors[arena[flat[i + 8], l]] = a if l != 0
+          l = flat[i + 11]
+          anchors[arena[flat[i + 10], l]] = a if l != 0
           merge_target.push(place(docs, stack, pending_key, pending_key_merge, a))
           stack.push(a)
           pending_key.push(nil)
           pending_key_merge.push(nil)
         when ALIAS
-          name = arena[flat[i + 6], flat[i + 7]]
+          name = arena[flat[i + 8], flat[i + 9]]
           raise Yeptris::ParseError, "unknown anchor: #{name.inspect}" unless anchors.key?(name)
 
           place(docs, stack, pending_key, pending_key_merge, anchors[name])

data/lib/yeptris/psych/handler.rb

@@ -49,6 +49,11 @@ module Yeptris
 
       def alias(anchor); end
 
+      # #179: called before EVERY event with the event's 0-based marks
+      # (psych 5's contract — TreeBuilder stashes them for node
+      # locations). No-op by default; handlers that care override it.
+      def event_location(start_line, start_column, end_line, end_column); end
+
       # +style+ is one of the scalar constants above; +plain+ /
       # +quoted+ are the implicit-typing flags (plain: resolvable
       # without a tag).

data/lib/yeptris/psych/parser.rb

@@ -52,12 +52,17 @@ module Yeptris
         h = @handler
         i = 0
         while i < flat.length
+          # #179: psych's parser calls event_location before every
+          # event; TreeBuilder stashes the marks for node locations
+          # (0-based).
+          h.event_location(flat[i + 4] - 1, flat[i + 5] - 1,
+                           flat[i + 6] - 1, flat[i + 7] - 1)
           type = flat[i]
           style = flat[i + 1]
           flags = flat[i + 2]
-          value = field(flat, arena, i + 6)
-          anchor = field(flat, arena, i + 8)
-          tag = field(flat, arena, i + 10)
+          value = field(flat, arena, i + 8)
+          anchor = field(flat, arena, i + 10)
+          tag = field(flat, arena, i + 12)
           case type
           when STREAM_START then h.start_stream(UTF8)
           when STREAM_END then h.end_stream

data/lib/yeptris/psych/tree_builder.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Yeptris
+  module Psych
+    # The stdlib TreeBuilder port (#179): builds a Nodes tree carrying
+    # start/end line:column on every node. The parser calls
+    # event_location before each event (psych 5's exact contract);
+    # stdlib's class is the reference — this port follows it
+    # function-for-function over the binding's keyword-arg Nodes.
+    class TreeBuilder < Handler
+      attr_reader :root
+
+      def initialize
+        super
+        @stack = []
+        @last = nil
+        @root = nil
+        @start_line = nil
+        @start_column = nil
+        @end_line = nil
+        @end_column = nil
+      end
+
+      def event_location(start_line, start_column, end_line, end_column)
+        @start_line = start_line
+        @start_column = start_column
+        @end_line = end_line
+        @end_column = end_column
+      end
+
+      %w[Sequence Mapping].each do |node|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def start_#{node.downcase}(anchor, tag, implicit, style)
+            n = Nodes::#{node}.new(anchor: anchor, tag: tag, style: style)
+            set_start_location(n)
+            @last.children << n
+            push n
+          end
+
+          def end_#{node.downcase}
+            n = pop
+            set_end_location(n)
+            n
+          end
+        RUBY
+      end
+
+      def start_document(version, tag_directives, implicit)
+        n = Nodes::Document.new(version, tag_directives)
+        set_start_location(n)
+        @last.children << n
+        push n
+      end
+
+      def end_document(_implicit = false)
+        n = pop
+        set_end_location(n)
+        n
+      end
+
+      def start_stream(_encoding)
+        n = Nodes::Stream.new
+        set_start_location(n)
+        @root = n
+        push n
+      end
+
+      def end_stream
+        n = pop
+        set_end_location(n)
+        n
+      end
+
+      def scalar(value, anchor, tag, plain, quoted, style)
+        n = Nodes::Scalar.new(value, anchor: anchor, tag: tag,
+                              plain: plain, quoted: quoted, style: style)
+        set_location(n)
+        @last.children << n
+        n
+      end
+
+      def alias(anchor)
+        n = Nodes::Alias.new(anchor)
+        set_location(n)
+        @last.children << n
+        n
+      end
+
+      private
+
+      def push(value)
+        @stack.push(value)
+        @last = value
+      end
+
+      def pop
+        value = @stack.pop
+        @last = @stack.last
+        value
+      end
+
+      def set_location(n)
+        n.start_line = @start_line
+        n.start_column = @start_column
+        n.end_line = @end_line
+        n.end_column = @end_column
+      end
+
+      def set_start_location(n)
+        set_location(n)
+      end
+
+      def set_end_location(n)
+        n.end_line = @end_line
+        n.end_column = @end_column
+      end
+    end
+  end
+end

data/lib/yeptris/psych.rb

@@ -86,6 +86,8 @@ module Yeptris
     # own entry so referencing Psych::Handlers triggers the load
     autoload :Handlers, "yeptris/psych/handler"
     autoload :Parser, "yeptris/psych/parser"
+    # #179: the located tree builder (stdlib's class, ported)
+    autoload :TreeBuilder, "yeptris/psych/tree_builder"
     autoload :CoderShim, "yeptris/psych/coder_shim"
     autoload :Visitors, "yeptris/psych/visitors"
     # The typed opt-in marker for arbitrary-object dump/load
@@ -425,6 +427,8 @@ module Yeptris
         include Enumerable
 
         attr_reader :children
+        # #179: the event marks (0-based), applied by TreeBuilder
+        attr_accessor :start_line, :start_column, :end_line, :end_column
         attr_reader :handle # @api private — the Yeptris::Node
         # @api private — the tree builder attaches handles; a writer,
         # never instance_variable_set from outside

data/lib/yeptris.rb

@@ -3,7 +3,7 @@
 module Yeptris
   # The gem's version lives in the parent namespace's file — the last
   # internal require (yeptris/version) retired with it.
-  VERSION = "0.6.18.2".freeze
+  VERSION = "0.6.20.1".freeze
   # The error hierarchy lives in THIS file (the parent namespace's
   # own file): nested constants do not trigger a parent-constant
   # autoload, and the law forbids internal requires — defining the
@@ -91,17 +91,21 @@ end
 # Optional C-API materializer (TODO.restructure/22): fused visit →
 # Ruby objects via the Ruby C API. Feature-detected — LoadError leaves
 # the FFI ladder (Marshal → columns → records) as the sole path.
-# Windows names the artifact per Ruby minor (the leptris-ruby #207/
-# #227 lesson): a PE DLL must bind its build Ruby's runtime, so one
-# DLL per supported minor ships and RUBY_VERSION selects. A missing
-# cell (Ruby 3.3 arm64 has no build) falls back LOUDLY to the ladder.
+# One bundle per Ruby minor (the leptris-ruby #207/#227 lesson: a PE
+# DLL must bind its build Ruby's runtime; a unix bundle is also
+# minor-locked by the internal API it compiles against). The MINOR
+# lives in the DIRECTORY (lib/yeptris/<minor>/native.<ext>) and the
+# FILE name stays unversioned: ruby derives the init symbol from the
+# feature basename, so native-3.4.so asked for the untypeable
+# "Init_native-3" and the versioned-file scheme could never load
+# (#157 — every shipped bundle was dead code). Dev checkouts keep
+# the plain native.<ext> name (mkmf layout, built for the running
+# Ruby). A missing cell falls back LOUDLY to the ladder.
 begin
-  # per-minor FIRST everywhere: a single version-agnostic native.so
-  # loads under ANY Ruby (symbols resolve from the host process), and
-  # one built for a different minor is an ABI gamble — Ruby 3.0 ran
-  # a 3.3-built ext and the JSON parse failed. The plain name stays
-  # as the dev-checkout fallback (built for the running Ruby).
-  require "yeptris/native-#{RUBY_VERSION[/\A\d+\.\d+/]}"
+  # per-minor FIRST: a single version-agnostic native.so loads under
+  # ANY Ruby and is an ABI gamble across minors — Ruby 3.0 ran a
+  # 3.3-built ext and the JSON parse failed.
+  require "yeptris/#{RUBY_VERSION[/\A\d+\.\d+/]}/native"
 rescue LoadError
   begin
     require "yeptris/native"

data/vendor/libyeptris/CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(yeptris
-    VERSION 0.6.18
+    VERSION 0.6.20
     DESCRIPTION "Ultra-fast YAML 1.2 parser, emitter and streamer in C"
     LANGUAGES C CXX
 )

data/vendor/libyeptris/src/CMakeLists.txt

@@ -24,6 +24,7 @@ set(YEPTRIS_SOURCES
     yeptris/parse/scalars.c
     yeptris/parse/engine.c
     yeptris/dom/dom.c
+    yeptris/dom/ytape.c
     yeptris/dom/hpool.c
     yeptris/dom/mapindex.c
     yeptris/dom/mutate.c

data/vendor/libyeptris/src/include/yeptris/events.h

@@ -66,6 +66,9 @@ typedef struct {
     size_t tag_len;
     uint32_t line; /* 1-based node start */
     uint32_t col;
+    uint32_t end_line; /* #179: libyaml end_mark semantics — 1-based line,
+                        * col one past the node's end token */
+    uint32_t end_col;
 } YeptrisEvent;
 
 /* ---- push ------------------------------------------------------------- */
@@ -106,13 +109,17 @@ typedef struct {
                      * struct's pad byte: sizeof stays 36 (ABI-pinned) */
     uint32_t line;
     uint32_t col;
+    uint32_t end_line; /* #179: libyaml end_mark semantics (see the
+                        * internal event) — 1-based, col past the end */
+    uint32_t end_col;
     uint32_t value_off;
     uint32_t value_len;
     uint32_t anchor_off;
     uint32_t anchor_len;
     uint32_t tag_off;
     uint32_t tag_len;
-} YeptrisEventRecord;
+} YeptrisEventRecord; /* 44B — the bindings mirror this layout in the
+                         same lockstep window (the 0.6.9 law) */
 
 YEPTRIS_API YeptrisRecorder yeptris_recorder_new(void);
 

data/vendor/libyeptris/src/yeptris/cbor/decode.c

@@ -868,6 +868,7 @@ static YeptrisDocument cbor_wrap(yep_dom* dom, const void* buf, const yep_alloca
     d->input = (const char*)buf;
     d->finish_pool = NULL;
     d->lazy_tape = NULL; /* field-by-field ctor: no garbage for free */
+    d->lazy_kind = 0;
     return (YeptrisDocument)d;
 }
 

data/vendor/libyeptris/src/yeptris/common/simd_text.h

@@ -99,6 +99,15 @@ typedef struct yep_chunk_masks {
     uint32_t valid;    /* 1-bits for the bytes that exist (tails) */
 } yep_chunk_masks;
 
+/* one 64-byte block's classification (json_stage1_masks) */
+struct yep_s1_block {
+    uint64_t q;      /* raw quote bytes */
+    uint64_t bs;     /* backslash bytes */
+    uint64_t op;     /* []{}:, */
+    uint64_t c0;     /* bytes < 0x20 (ws included) */
+    uint64_t tokens; /* the resolver's verdict: ops + string-open quotes */
+};
+
 /* The kernel table. One struct = one dispatch point (OCP: a new ISA is a
  * new TU exporting a new table; nothing else changes). */
 typedef struct yep_text_kernels {
@@ -175,10 +184,25 @@ typedef struct yep_text_kernels {
      * hard error is an unterminated string (odd final parity) ->
      * return 0. idx must hold len+2 entries. Bit-identical across
      * ISAs (the differential suite pins it); scan/json.c owns the
-     * scalar reference and the grammar contract. */
-    int (*json_stage1)(const char* p, size_t len, uint32_t* idx, size_t* nidx);
+     * grammar contract. flags (never NULL) accumulates YEP_S1_* bits:
+     * the doc-level facts stage 2 needs to skip content scanning. */
+    int (*json_stage1)(const char* p, size_t len, uint32_t* idx, size_t* nidx, unsigned* flags);
+
+    /* The structural indexer's classification half, without the
+     * emission: per 64-byte block, the four u64 masks the resolver
+     * consumes (QUOTE/BS/OP/C0; ws is dead since the token contract
+     * dropped scalar-run starts). blocks receives len/64+1 records;
+     * *nblocks returns how many were written. Same reject/flag
+     * contract as json_stage1. The single-pass tape walk consumes
+     * this — no idx array, no emission loop. */
+    int (*json_stage1_masks)(const char* p, size_t len, struct yep_s1_block* blocks,
+                             size_t* nblocks, unsigned* flags);
 } yep_text_kernels;
 
+/* json_stage1 flag bits (accumulated across blocks; cleared by caller) */
+#define YEP_S1_C0_IN_STRING 1u /* raw non-ws C0 inside a string: reject */
+#define YEP_S1_HAS_ESCAPE 2u   /* a backslash byte exists: validate spans */
+
 /* The structural indexer's mask resolver — the SSOT identity set for
  * every json_stage1 implementation (scalar in scan/json.c, the ISA
  * twins in the SIMD TUs). Given one 64-byte block's QUOTE/BS/OP/WS
@@ -195,15 +219,13 @@ typedef struct yep_text_kernels {
  *   tokens     = (op | starts) & ~string_tail
  *
  * Returns the block's contribution; carries update in place. */
-static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t op, uint64_t ws,
-                                             uint64_t valid, uint64_t* prev_in_string,
-                                             uint64_t* esc_carry, uint64_t* follows_carry,
-                                             size_t off, uint32_t* idx, size_t n) {
+static inline uint64_t yep_json_stage1_tokens(uint64_t q, uint64_t bs, uint64_t op, uint64_t c0,
+                                              uint64_t valid, uint64_t* prev_in_string,
+                                              uint64_t* esc_carry, unsigned* flags) {
     uint64_t potential = bs & ~*esc_carry;
     uint64_t s = (((potential << 1) | 0xAAAAAAAAAAAAAAAAull) - potential) ^ 0xAAAAAAAAAAAAAAAAull;
-    uint64_t esc_bs = s & bs;
     uint64_t escaped = s ^ (bs | *esc_carry);
-    *esc_carry = esc_bs >> 63;
+    *esc_carry = (s & bs) >> 63;
     uint64_t quote_live = q & ~escaped;
     uint64_t ps = quote_live; /* sequential prefix-xor — a flat
                                * one-expression xor of shifts misses
@@ -217,15 +239,27 @@ static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t o
     uint64_t in_string = ps ^ *prev_in_string;
     *prev_in_string = (uint64_t)((int64_t)in_string >> 63);
     uint64_t string_tail = (in_string ^ quote_live) & valid;
-    uint64_t scalar_nq = (~(op | ws) & ~q) & valid; /* non-quote scalars */
-    uint64_t follows = ((scalar_nq & ~(1ull << 63)) << 1) | *follows_carry;
-    *follows_carry = scalar_nq >> 63;
-    uint64_t tokens = ((op | (~(op | ws) & ~follows)) & ~string_tail) & valid;
+    if (bs != 0) {
+        *flags |= YEP_S1_HAS_ESCAPE;
+    }
+    if ((c0 & in_string & valid) != 0) {
+        *flags |= YEP_S1_C0_IN_STRING;
+    }
+    return (op | (quote_live & ~string_tail)) & ~string_tail & valid;
+}
+
+/* The full resolver: token mask derivation + the idx emission loop
+ * (the json_stage1 implementations). */
+static inline size_t yep_json_stage1_resolve(uint64_t q, uint64_t bs, uint64_t op, uint64_t ws,
+                                             uint64_t c0, uint64_t valid, uint64_t* prev_in_string,
+                                             uint64_t* esc_carry, size_t off, uint32_t* idx,
+                                             size_t n, unsigned* flags) {
+    uint64_t tokens =
+        yep_json_stage1_tokens(q, bs, op, c0, valid, prev_in_string, esc_carry, flags);
     while (tokens != 0) {
         idx[n++] = (uint32_t)(off + (size_t)yep_ctz64(tokens));
         tokens &= tokens - 1;
     }
-    (void)escaped;
     return n;
 }
 

data/vendor/libyeptris/src/yeptris/common/simd_text_avx2.c

@@ -455,12 +455,13 @@ yep_chunk_masks yep_text_json_chunk_avx2(const char* p, size_t n) {
  * (two 32-byte lane passes each, movemask-native), then the shared
  * mask resolver (simd_text.h). The final partial block builds its
  * masks byte-wise — no loads past len. */
-int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t* nidx) {
+int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                              unsigned* flags) {
     size_t n = 0;
-    uint64_t prev_in_string = 0, esc_carry = 0, follows_carry = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
     size_t off = 0;
     for (; off + 64 <= len; off += 64) {
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (unsigned half = 0; half < 2; half++) {
             const __m256i v =
                 _mm256_loadu_si256((const __m256i*)(const void*)(p + off + 32 * half));
@@ -482,17 +483,21 @@ int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t*
                                 _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\n')),
                                                 _mm256_cmpeq_epi8(v, _mm256_set1_epi8('\r'))));
             uint64_t fw = (uint64_t)(uint32_t)_mm256_movemask_epi8(w);
+            /* unsigned min(v, 0x20) == v  <=>  v < 0x20 */
+            uint64_t fc = (uint64_t)(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+                _mm256_max_epu8(v, _mm256_set1_epi8(0x1F)), _mm256_set1_epi8(0x1F)));
             q |= fq << (32 * half);
             bs |= fb << (32 * half);
             op |= fo << (32 * half);
             ws |= fw << (32 * half);
+            c0 |= fc << (32 * half);
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, ~0ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, ~0ull, &prev_in_string, &esc_carry, off, idx,
+                                    n, flags);
     }
     if (off < len) {
         size_t cn = len - off;
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (size_t k = 0; k < cn; k++) {
             unsigned char c = (unsigned char)p[off + k];
             uint64_t bit = 1ull << k;
@@ -505,14 +510,86 @@ int yep_text_json_stage1_avx2(const char* p, size_t len, uint32_t* idx, size_t*
             } else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
                 ws |= bit;
             }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
         }
-        n = yep_json_stage1_resolve(q, bs, op, ws, (1ull << cn) - 1ull, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, (1ull << cn) - 1ull, &prev_in_string,
+                                    &esc_carry, off, idx, n, flags);
     }
     *nidx = n;
     return prev_in_string ? 0 : 1;
 }
 
+/* The classification half of the structural indexer (AVX2 twin of the
+ * NEON masks slot): two 32-byte lane passes per 64-byte block. */
+int yep_text_json_stage1_masks_avx2(const char* p, size_t len, struct yep_s1_block* blocks,
+                                    size_t* nblocks, unsigned* flags) {
+    size_t nb = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
+    size_t off = 0;
+    for (; off + 64 <= len; off += 64) {
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (unsigned half = 0; half < 2; half++) {
+            const __m256i v =
+                _mm256_loadu_si256((const __m256i*)(const void*)(p + off + 32 * half));
+            uint64_t fq = (uint64_t)(uint32_t)_mm256_movemask_epi8(
+                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"')));
+            uint64_t fb = (uint64_t)(uint32_t)_mm256_movemask_epi8(
+                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\')));
+            __m256i o = _mm256_or_si256(
+                _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('{')),
+                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8('}'))),
+                _mm256_or_si256(_mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('[')),
+                                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8(']'))),
+                                _mm256_or_si256(_mm256_cmpeq_epi8(v, _mm256_set1_epi8(',')),
+                                                _mm256_cmpeq_epi8(v, _mm256_set1_epi8(':')))));
+            uint64_t fo = (uint64_t)(uint32_t)_mm256_movemask_epi8(o);
+            uint64_t fc = (uint64_t)(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+                _mm256_max_epu8(v, _mm256_set1_epi8(0x1F)), _mm256_set1_epi8(0x1F)));
+            q |= fq << (32 * half);
+            bs |= fb << (32 * half);
+            op |= fo << (32 * half);
+            c0 |= fc << (32 * half);
+        }
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, ~0ull, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    if (off < len) {
+        size_t cn = len - off;
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (size_t k = 0; k < cn; k++) {
+            unsigned char c = (unsigned char)p[off + k];
+            uint64_t bit = 1ull << k;
+            if (c == '"') {
+                q |= bit;
+            } else if (c == '\\') {
+                bs |= bit;
+            } else if (c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == ':') {
+                op |= bit;
+            }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
+        }
+        uint64_t valid = (1ull << cn) - 1ull;
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, valid, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    *nblocks = nb;
+    return prev_in_string ? 0 : 1;
+}
+
 const yep_text_kernels yep_text_kernels_avx2 = {
     yep_avx2_contains,
     yep_avx2_find,
@@ -529,6 +606,7 @@ const yep_text_kernels yep_text_kernels_avx2 = {
     yep_avx2_line_facts,
     yep_text_json_chunk_avx2,
     yep_text_json_stage1_avx2,
+    yep_text_json_stage1_masks_avx2,
 };
 
 #endif /* YEP_ARCH_X86 */