yeptris 0.6.32.1-aarch64-linux → 0.6.33.2-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27054940f25d07b0af5eebd56ff85e79de42305a7ab59e34f714b48ab523458c
-  data.tar.gz: aef30cec9183a9caf10864e3d112d109bf3c81f93000c11c87d5e4547b22ba50
+  metadata.gz: 4f539306a31099d38b0040d34986abe9aecaf7f4f7950f78fbd828002d0d186d
+  data.tar.gz: a325b02fec0989a9d21ee8615c7993d05e9a9b2f7062a64560656a4699160109
 SHA512:
-  metadata.gz: 5567d722a33e0f30f60ce64345be3c2ee57657bdd884b82f159f6ec9d5ec415ce6c83cd70ae317d3449e13824fd5a018b1e3be7c62440e02832c6c2b9bf479cf
-  data.tar.gz: 1e178a8fe0eda68348d962919ed6523c0347cc851de265804307a1d09aa2c5c80cc2859452d19b4c0fe1c14463184974d017d0000220db234d135905b5ada08c
+  metadata.gz: 0ac10a304643b6fa2dd77979974b0809c6032c8918a9c9a9e192667e479338f0b037a45f1c240e20dccb110f354d46854c3cde5414d14a861a9577b353baba83
+  data.tar.gz: 1ba4bb2d25f6c2cbca95360336b8a8827533dace201394f8f92c2cea6395743bee00613963aa650ea2e9cafdc4ff647e7a5db4b251b25086a85491bfee6c6349

data/README.adoc

@@ -5,6 +5,15 @@ https://github.com/leptris/yeptris[libyeptris] — the YAML counterpart
 of libleptris. Psych-compatible semantics, one shared library, zero
 compilation at install.
 
+Part of the yeptris program: *the best (multiple dimensions)
+YAML/JSON parser/builder ever* — conformance, performance,
+safety/hardening, API surfaces, transport correctness, and trust,
+held simultaneously, in C, Ruby, Rust, and JS. This binding is the
+most complete surface: the Psych drop-in namespace, JSON.parse
+parity with the strict duplicate-key contract, CBOR, and ValueML,
+tracking every libyeptris release in lockstep. The achievement
+program lives in the engine's PLAN.md ("The best-ever program").
+
 == Install (development)
 
 The gem loads a shared `libyeptris` — from `YEPTRIS_LIB_PATH`, a

data/ext/yeptris_native/json_ruby.c

@@ -173,9 +173,19 @@ static VALUE jr_object(jr* j) {
 }
 
 static VALUE jr_object_body(jr* j, VALUE h_pre) {
+    /* The pairs staging buffer MUST live in GC-scanned memory: pairs
+     * beyond the current one exist only there once the key/val locals
+     * are reused, so a malloc'd buffer makes them invisible to the
+     * collector — it can free them mid-parse and recycle their slots,
+     * and rb_hash_bulk_insert then inserts garbage (yeptris-ruby#279:
+     * the arm64-darwin SIGSEGV whose fault address was document text).
+     * The stack array IS scanned, so the bulk fast path is bounded at
+     * 31 pairs; beyond that the buffered pairs flush with one
+     * bulk_insert and the rest of the object rides rb_hash_aset. */
     VALUE pairs[64];
     VALUE* pv = pairs;
     size_t pcap = 64, pn = 0, heap_cap = 0;
+    int aset_rest = 0;
     for (;;) {
         jr_ws(j);
         if (j->i >= j->len || j->p[j->i] != '"') { j->err = -2; goto out; }
@@ -186,7 +196,7 @@ static VALUE jr_object_body(jr* j, VALUE h_pre) {
         j->i++;
         VALUE val = jr_value(j);
         if (j->err) goto out;
-        if (yep_ins_mode == YEP_INS_ASET || j->strict_dup) {
+        if (yep_ins_mode == YEP_INS_ASET || j->strict_dup || aset_rest) {
             VALUE h = h_pre == Qundef ? (h_pre = HASH_NEW_CAPA(8)) : h_pre;
             if (j->strict_dup && !NIL_P(rb_hash_aref(h, key))) {
                 j->err = -3;
@@ -196,15 +206,21 @@ static VALUE jr_object_body(jr* j, VALUE h_pre) {
             rb_hash_aset(h, key, val);
         } else {
             if (pn + 2 > pcap) {
-                size_t ncap = pcap * 2;
-                VALUE* nv = malloc(ncap * sizeof(VALUE));
-                if (!nv) { j->err = -1; goto out; }
-                memcpy(nv, pv, pn * sizeof(VALUE));
-                if (pv != pairs) { free(pv); heap_cap = 1; }
-                pv = nv; pcap = ncap;
+                /* buffer full: flush what is staged (no heap staging —
+                 * see the #279 note above); THIS pair and the rest of
+                 * the object ride aset — no continue here, the
+                 * delimiter scan below must still run */
+                VALUE h = (h_pre == Qundef) ? (h_pre = HASH_NEW_CAPA((long)(pn / 2) + 8)) : h_pre;
+                rb_hash_bulk_insert((long)pn, (const VALUE*)pv, h);
+                pn = 0;
+                aset_rest = 1;
+            }
+            if (aset_rest) {
+                rb_hash_aset(h_pre, key, val);
+            } else {
+                pv[pn++] = key;
+                pv[pn++] = val;
             }
-            pv[pn++] = key;
-            pv[pn++] = val;
         }
         jr_ws(j);
         if (j->i >= j->len) { j->err = -2; goto out; }
@@ -351,7 +367,17 @@ double yep_rb_scan_time(const char* p, size_t len, int n) {
 VALUE yep_rb_parse_json(const char* p, size_t len, int strict_dup) {
     jr j;
     memset(&j, 0, sizeof(j));
-    j.p = p; j.len = len; j.enc = rb_utf8_encoding();
+    /* The parse holds j.p across unbounded allocations, and the GC
+     * strategy knobs leave collection ENABLED during the window
+     * (arm64's default is none). A compaction moving the source
+     * string would dangle every j->p read after the move
+     * (yeptris-ruby#279's heap-layout-dependent SIGSEGV class), so
+     * the parse runs on a private copy: one memcpy of the input,
+     * then no Ruby object backs the cursor. */
+    char* own = malloc(len ? len : 1);
+    if (own == NULL) rb_raise(rb_eNoMemError, "yeptris native json");
+    memcpy(own, p, len);
+    j.p = own; j.len = len; j.enc = rb_utf8_encoding();
     j.strict_dup = strict_dup;
     VALUE already = Qtrue;
     if (yep_gc_mode != YEP_GC_NONE) already = rb_gc_disable();
@@ -359,6 +385,7 @@ VALUE yep_rb_parse_json(const char* p, size_t len, int strict_dup) {
     jr_ws(&j);
     if (j.i != len && j.err == 0) j.err = -2;
     free(j.scratch);
+    free(own);
     if (already == Qfalse) rb_gc_enable();
     if (yep_gc_mode == YEP_GC_START && j.err == 0) rb_gc_start();
     if (j.err == -1) rb_raise(rb_eNoMemError, "yeptris native json");
@@ -366,6 +393,11 @@ VALUE yep_rb_parse_json(const char* p, size_t len, int strict_dup) {
         rb_raise(rb_path2class("Yeptris::ParseError"), "duplicate key \"%s\" in JSON object",
                  RSTRING_PTR(j.dup_key));
     }
-    if (j.err != 0) rb_raise(rb_path2class("Yeptris::ParseError"), "native json parse failed");
+    if (j.err != 0) {
+        /* the tape engine's wording ("not strict JSON at byte N") so
+         * consumers can pin one error shape across both routes
+         * (yeptris-ruby#279's adjacent note) */
+        rb_raise(rb_path2class("Yeptris::ParseError"), "not strict JSON at byte %zu", j.i);
+    }
     return v;
 }

data/lib/yeptris/ffi.rb

@@ -228,12 +228,17 @@ module Yeptris
                     %i[pointer size_t int int pointer pointer], :int
     attach_function :yeptris_marshal_node,
                     %i[yeptris_node pointer pointer], :int
+    attach_function :yeptris_marshal_node_ex,
+                    %i[yeptris_node uint32 pointer pointer], :int
     attach_function :yeptris_marshal_free, [:pointer], :void
     # The bulk child drain (#168's quadratic): O(n) iteration for
     # #each/#each_pair. Engines without it fall back to the per-index
     # walk (correct, quadratic).
     CHILDREN_DRAIN = !(@missing ||= []).include?(:yeptris_node_children)
     MARSHAL = !(@missing ||= []).include?(:yeptris_marshal_node)
+    # the strict-dup marshal entry (engine 0.6.34+, yeptris-ruby#278):
+    # gates the JSON fallback's C route in STRICT mode
+    MARSHAL_STRICT_DUP = !(@missing ||= []).include?(:yeptris_marshal_node_ex)
 
     # bulk build (TODO.impl/15 phase D): one call raises a document
     BUILD_SCALAR = 1

data/lib/yeptris/json.rb

@@ -9,10 +9,13 @@ module Yeptris
   # keeps the Psych contract for every input (JSON-shaped included),
   # so the two never drift into each other's semantics.
   #
-  # Engines, fastest first (both exact — the parity spec runs against
+  # Engines, fastest first (all exact — the parity spec runs against
   # whichever is loaded):
   # 1. the native materializer (fused C scan → VALUE; opt-in build)
-  # 2. the record drain + strict conversion walk (always available)
+  # 2. the marshal engine (#278): strict-JSON parse + Marshal 4.8
+  #    emission + ONE core Marshal.load — all C, no per-value Ruby
+  #    (engine 0.6.34+; the strict flag follows the resolved json gem)
+  # 3. the record drain + strict conversion walk (always available)
   module JSON
     class Error < ::Yeptris::Error; end
     class ParseError < Error; end
@@ -39,9 +42,46 @@ module Yeptris
           raise ParseError, e.message
         end
       end
+      # yeptris-ruby#278: with no Ruby-version-matched native build the
+      # pure-Ruby walk measured 0.11x vs JSON.parse on real documents —
+      # the marshal engine keeps the fallback C-side end to end.
+      return marshal_engine(source, 0) if ::Yeptris::FFI::MARSHAL_STRICT_DUP
       tape_engine(source)
     end
 
+    # The marshal engine (#278): yeptris_parse_json gates the strict
+    # grammar, yeptris_marshal_node_ex writes the Ruby Marshal 4.8
+    # stream (STRICT_DUP_KEYS raising on json 3's default contract),
+    # and ONE core Marshal.load materializes the graph. Returns nil
+    # when the engine cannot serve the route (caller keeps walking);
+    # every parse/dup failure raises ParseError.
+    def marshal_engine(source, flags)
+      begin
+        gate = ::Yeptris::Document.parse_json(source)
+      rescue ::Yeptris::ParseError => e
+        raise ParseError, e.message
+      end
+      begin
+        out_p = ::FFI::MemoryPointer.new(:pointer)
+        olen_p = ::FFI::MemoryPointer.new(:size_t)
+        st = ::Yeptris::FFI.yeptris_marshal_node_ex(
+          ::Yeptris::FFI.yeptris_document_root(gate.c_ptr, 0), flags, out_p, olen_p
+        )
+        if st != ::Yeptris::FFI::OK
+          raise ParseError, ::Yeptris::FFI.last_error_message if st == ::Yeptris::FFI::ERROR_PARSE
+
+          return nil # UNSUPPORTED: merge keys et al. cannot occur in JSON
+        end
+        buf = out_p.read_pointer
+        data = buf.read_bytes(::Yeptris::FFI.read_c_size_t(olen_p))
+        data.force_encoding(Encoding::ASCII_8BIT)
+        ::Yeptris::FFI.yeptris_marshal_free(buf)
+        Marshal.load(data)
+      ensure
+        gate.free
+      end
+    end
+
     # The tape engine (issue #81): ONE call validates and materializes
     # the record stream; the walk reads four bulk columns and slices
     # strings straight out of the caller's source — no gate document,
@@ -212,6 +252,12 @@ module Yeptris
         raise ParseError, e.message
       end
       begin
+        if ::Yeptris::FFI::MARSHAL_STRICT_DUP
+          routed = marshal_engine(
+            source, STRICT_DUPLICATE_KEYS ? ::Yeptris::FFI::MARSHAL_STRICT_DUP_KEYS : 0
+          )
+          return routed unless routed.nil?
+        end
         cols = ::Yeptris::FFI::ValueColumns.new
         st = ::Yeptris::FFI.yeptris_value_drain_columns(
           source, source.bytesize, ::Yeptris::FFI::SCHEMA_12_CORE, cols

data/lib/yeptris/node.rb

@@ -50,7 +50,7 @@ class Yeptris::Node
 
   # Scalar content / alias name (UTF-8 String), nil for collections.
   def value
-    len = ::FFI::MemoryPointer.new(:size_t)
+    len = len_buffer
     ptr = alive { Yeptris::FFI.yeptris_node_value(@c_ptr, len) }
     return nil if ptr.null?
 
@@ -69,9 +69,18 @@ class Yeptris::Node
     STYLES[alive { Yeptris::FFI.yeptris_node_style(@c_ptr) }]
   end
 
+  # The out-length receiver: ONE buffer per thread, reused. A fresh
+  # FFI::MemoryPointer per accessor call was an allocation storm that
+  # stalled the Windows heap under the #182 churn loop (the #504
+  # family — the hang's stack sits in FFI::MemoryPointer#initialize,
+  # node.rb anchor/tag reads inside Integer#times with GC.start).
+  def len_buffer
+    Thread.current[:yeptris_len_buf] ||= ::FFI::MemoryPointer.new(:size_t)
+  end
+
   # Explicit tag URI when present, else nil.
   def tag
-    len = ::FFI::MemoryPointer.new(:size_t)
+    len = len_buffer
     ptr = alive { Yeptris::FFI.yeptris_node_tag(@c_ptr, len) }
     return nil if ptr.null?
 
@@ -79,7 +88,7 @@ class Yeptris::Node
   end
 
   def anchor
-    len = ::FFI::MemoryPointer.new(:size_t)
+    len = len_buffer
     ptr = alive { Yeptris::FFI.yeptris_node_anchor(@c_ptr, len) }
     return nil if ptr.null?
 

data/lib/yeptris/psych/visitors.rb

@@ -216,7 +216,7 @@ module Yeptris
               # explicit !!str tag (single-quoted) so it does not
               # re-load as a merge
               key_node = @tree.new_scalar("<<", :single_quoted)
-              key_node.set_tag("!!str")
+              key_node.set_tag("tag:yaml.org,2002:str")
               m.map_add_node(key_node, visit(v))
             else
               m.map_add(key_text(k), visit(v))

data/lib/yeptris/yaml.rb

@@ -364,7 +364,7 @@ module Yeptris
               # stdlib yaml_tree: a literal chevron key carries the
               # explicit !!str tag so it does not re-load as a merge
               key_node = doc.new_scalar("<<", :single_quoted)
-              key_node.set_tag("!!str")
+              key_node.set_tag("tag:yaml.org,2002:str")
               m.map_add_node(key_node, build(doc, v, seen))
             else
               m.map_add(key_text(k), build(doc, v, seen))

data/lib/yeptris.rb

@@ -3,7 +3,7 @@
 module Yeptris
   # The gem's version lives in the parent namespace's file — the last
   # internal require (yeptris/version) retired with it.
-  VERSION = "0.6.32.1".freeze
+  VERSION = "0.6.33.2".freeze
 
   # Informational notices are OPT-IN (yeptris-ruby#217): consumers
   # embed yeptris inside CLIs that assert clean stderr, and a

data/vendor/libyeptris/CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(yeptris
-    VERSION 0.6.32
+    VERSION 0.6.33
     DESCRIPTION "Ultra-fast YAML 1.2 parser, emitter and streamer in C"
     LANGUAGES C CXX
 )
@@ -98,15 +98,27 @@ if(YEPTRIS_ENABLE_UBSAN)
     message(STATUS "UndefinedBehaviorSanitizer: ENABLED (fatal)")
 endif()
 
-# libFuzzer implies ASAN (libleptris TODO 40 pattern).
+# libFuzzer implies ASAN+UBSan (the teptris fuzz.yml pattern: the
+# memory-safety contracts ride the SAME binary as the fuzzer, so every
+# nightly round reports UB at the crash site instead of libFuzzer's
+# "combine with AddressSanitizer" note).
 if(YEPTRIS_ENABLE_FUZZING)
     # instrument everything, but link the fuzzer MAIN only into the
     # harnesses: a global -fsanitize=fuzzer link option demands
     # LLVMFuzzerTestOneInput from every executable (the CLI and the
     # test binaries died on it once CXX also pinned to clang++)
-    add_compile_options(-fsanitize=fuzzer-no-link,address -fno-omit-frame-pointer -g)
-    add_link_options(-fsanitize=address)
-    message(STATUS "libFuzzer: ENABLED (implies ASAN)")
+    if(APPLE)
+        # Apple ld64 cannot link UBSan'd static archives
+        # ("invalid r_symbolnum"): ASan-only locally
+        add_compile_options(-fsanitize=fuzzer-no-link,address -fno-omit-frame-pointer -g)
+        add_link_options(-fsanitize=address)
+        message(STATUS "libFuzzer: ENABLED (implies ASAN; UBSan needs linux ld)")
+    else()
+        add_compile_options(-fsanitize=fuzzer-no-link,address,undefined
+            -fno-sanitize-recover=undefined -fno-omit-frame-pointer -g)
+        add_link_options(-fsanitize=address,undefined)
+        message(STATUS "libFuzzer: ENABLED (implies ASAN+UBSan)")
+    endif()
 endif()
 
 # -------------------------------------------------------------------- LTO --

data/vendor/libyeptris/src/yeptris/dom/dom.c

@@ -392,6 +392,27 @@ int yep_dom_on_event(void* ctx, const yep_event* ev) {
         if (d->depth == 0) {
             return -1;
         }
+        if (ev->type == YEP_EV_MAP_END && d->map_pending_key[d->depth - 1]) {
+            /* an odd child reached the mapping: the unpaired node —
+             * possibly an ANCHORED one — used to be dropped here, its
+             * anchor lost, and a later alias then emitted a dangling
+             * reference the re-parse rejected (the nightly's round
+             * 102). Pair it with a null: "k:" with nothing is a null
+             * value in every other position. */
+            yep_event nv;
+            memset(&nv, 0, sizeof(nv));
+            nv.type = YEP_EV_SCALAR;
+            nv.style = YEP_STYLE_PLAIN;
+            nv.implicit = 1;
+            uint32_t nid = dom_new_node(d, &nv, YEP_DOM_SCALAR);
+            if (nid == UINT32_MAX) {
+                return -1;
+            }
+            uint32_t top = d->stack[d->depth - 1];
+            dom_link(d, top, d->pending_key_id[d->depth - 1]);
+            dom_link(d, top, nid);
+            d->map_pending_key[d->depth - 1] = 0;
+        }
         d->depth--;
         return 0;
 
@@ -729,6 +750,7 @@ static int jb_value(jbuilder* b) {
             return 0;
         }
         for (;;) {
+            size_t member_start = b->i;
             int rc = jb_value(b);
             if (rc != 0) {
                 return rc;
@@ -738,6 +760,14 @@ static int jb_value(jbuilder* b) {
                 /* a value must be followed by , or } — the map's key
                  * slot is pending until the ':' value completes */
                 if (b->i < b->len && b->p[b->i] == ':' && d->map_pending_key[d->depth - 1] != 0) {
+                    /* libyaml's simple-key law rides the JSON-class
+                     * route too: a key whose colon distance crosses
+                     * 1024 characters defers to the engine's clean
+                     * error (run 38143486175) */
+                    if (b->i - member_start > 1024 &&
+                        yep_simple_key_chars(b->p + member_start, b->i - member_start) > 1024) {
+                        return -2;
+                    }
                     b->i++;
                     rc = jb_value(b);
                     if (rc != 0) {
@@ -820,8 +850,11 @@ int dom_on_flow_build(void* ctx, const char* p, size_t open, size_t len, uint32_
     yep_json_walk w;
     yep_json_tok t;
     yep_json_walk_init(&w, p, len, open, max_depth);
-    w.strict = d->flow_strict; /* the strict-JSON route rides the same
-                                  fused walk (TODO.restructure/81) */
+    w.strict = 1; /* the same byte-for-byte JSON gate as the engine's
+                   * pass 1: a lenient walk split number/literal keys
+                   * across the fast/slow paths (whole {9->1} vs chunked
+                   * {"9:1"->null}); flow_strict keeps owning the TAG
+                   * route only (TODO.restructure/81) */
     for (;;) {
         yep_jw_status st = yep_json_walk_next(&w, &t);
         if (st == YEP_JW_REJECT) {
@@ -976,6 +1009,23 @@ int dom_from_tape(yep_dom* d, const yeptris_json_tape* t) {
             if (d->depth == 0) {
                 return -1;
             }
+            if (d->nodes[d->stack[d->depth - 1]].kind == YEP_DOM_MAPPING &&
+                d->map_pending_key[d->depth - 1]) {
+                /* an odd child reached the mapping (see the event-path
+                 * MAP_END): pair it with a null instead of dropping it */
+                yep_event nv;
+                memset(&nv, 0, sizeof(nv));
+                nv.type = YEP_EV_SCALAR;
+                nv.style = YEP_STYLE_PLAIN;
+                nv.implicit = 1;
+                uint32_t nid = dom_new_node(d, &nv, YEP_DOM_SCALAR);
+                if (nid == UINT32_MAX) {
+                    return -1;
+                }
+                dom_link(d, d->stack[d->depth - 1], d->pending_key_id[d->depth - 1]);
+                dom_link(d, d->stack[d->depth - 1], nid);
+                d->map_pending_key[d->depth - 1] = 0;
+            }
             d->depth--;
             break;
         case YEP_T_NULL: