yeptris 0.1.12.0 → 0.1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fa74d905a2d19dbeaf70381760c3a629738808e507d0ef9f33c9e266b756854
-  data.tar.gz: 2d3fb1c11b0d2dbb88c5bb75e837f41abe971992c1940a8c5be191b0168411b8
+  metadata.gz: 76bc800fd5065540397d957ca2162d2491474a086eda5b08436f016daa8f65e4
+  data.tar.gz: 683259ac45cfb084722337af80c066890bdf56f2608843343ebf8951589eafa8
 SHA512:
-  metadata.gz: e621ced9cf2707ada102a8633f7f75f597f63488f2a66da4efdb0dd3e292ef0719bbba509db321ca9e5cfd38f455a732a90cb3d1709b79f4ae7cab5ef0dd48da
-  data.tar.gz: 9d5efd07ce2e2f498320133776e3b0ed08e1fec4977a7e5cd1b607616630377bc29f7d5708e00c058c2f7d97022a6b668d6e3865c33da17a7d9c8ba71d445f2e
+  metadata.gz: 3b2be1bc057ee5a059e80ac087000afd2cee325473d834db34bd0f898318634e7b9be95dce9b523f909ae6884a56c9ede549a0e7142184eddf4cc96a13a11cb9
+  data.tar.gz: e95683f3b475a0dca70488a43cdf23a249d2bcd49c440b4c75bf31914265c0eae9fd2992d1bf6c0ebfa5b0487b796f1a24505e42b4715840c1731ce2af845a4c

data/README.adoc

@@ -52,9 +52,37 @@ Handles are document-scoped: `Document#free` releases everything
 (one C call), a GC finalizer backs it up, and any use after free
 raises `Yeptris::FreedError` — never a segfault.
 
-== Native materializer (opt-in): faster than JSON.parse
+== Two load surfaces: YAML and strict JSON
+
+`Yeptris::YAML.load` keeps the Psych contract for EVERY input —
+including JSON-shaped ones. `{"a": [1,]}` is legal flow YAML (spec
+production [141] allows the trailing comma); `"1e3"` is a Psych
+String. These semantics never flip because input happens to look
+like JSON.
+
+`Yeptris::JSON.load` is the STRICT RFC 8259 surface — exact
+`JSON.parse` semantics by construction (spec-pinned in
+`spec/json_parity_spec.rb`: every value and every error case,
+Bignum integers, exponent-only floats, duplicate keys). Engines,
+fastest first:
+
+1. **Native materializer** (opt-in build; see below): a fused C scan
+   → `VALUE` parser. Order-alternating interleaved profile, 152 KB /
+   ~29k-value corpus, Ruby 3.4:
++
+....
+JSON.parse        min 0.670  med 0.966  mean 0.910 ms
+Yeptris::JSON     min 0.637  med 0.674  mean 0.680 ms
+                  mean 0.75x faster, head-to-head 370/400 (92.5%)
+....
+2. **Record-drain fallback** (always available): the strict-JSON
+   validator gates, then the value records convert without the Psych
+   quirk table — exact parity with engine 1, spec-pinned.
 
-For JSON-shaped input, the binding can beat Ruby's own `JSON.parse`:
+The committed profile (`benchmark/json_profile.rb`) is the fair
+benchmark of record — any performance claim runs through it.
+
+=== Building the native materializer (opt-in)
 
 ....
 cd ext/yeptris_native
@@ -62,22 +90,9 @@ YEPTRIS_LIB_PATH=/path/to/libyeptris.dylib ruby extconf.rb && make
 cp native.bundle ../../lib/yeptris/   # or .so on Linux
 ....
 
-`require "yeptris"` picks it up automatically (a missing build falls
-back to the FFI Marshal ladder silently). Measured on the 152 KB /
-29.4k-value JSON corpus (mean of 300, Ruby 3.4):
-
-....
-JSON.parse           mean 1.597 ms
-Yeptris::YAML.load   mean 1.145 ms   (0.72x — faster than JSON.parse)
-Psych.load           mean 40.60 ms   (35x slower than yeptris)
-....
-
-The extension is a fused RFC 8259 → `VALUE` parser (one pass, no
-intermediate records): libyeptris scan kernels tokenize, the Ruby C
-API allocates, repeated keys/tokens share one frozen `String`, and GC
-is paused for the duration. YAML inputs keep the FFI ladder so
-timestamps, aliases, and Psych's scalar quirks resolve through the
-same path `Psych.load` uses.
+`require "yeptris"` picks it up automatically (`Yeptris::JSON` then
+uses it; missing builds silently use the fallback — the gem installs
+without compiling).
 
 == Shipped beyond the original plan
 

data/ext/yeptris_native/extconf.rb

@@ -2,6 +2,9 @@
 
 require "mkmf"
 
+# libyeptris location: YEPTRIS_LIB_PATH (file or dir), then sibling
+# checkouts. CI sets YEPTRIS_LIB_PATH (the built shared library) and
+# YEPTRIS_SRC (the C checkout) explicitly.
 lib_path = ENV["YEPTRIS_LIB_PATH"]
 candidates = []
 if lib_path
@@ -13,11 +16,12 @@ candidates << File.expand_path("../../../../yeptris/build/src", __dir__)
 candidates << File.expand_path("../../../yeptris/build-validate/src", __dir__)
 candidates << File.expand_path("../../../yeptris/build/src", __dir__)
 
-src_root = [
-  File.expand_path("../../../../yeptris/src", __dir__),
-  File.expand_path("../../../yeptris/src", __dir__),
-].find { |d| d && File.directory?(File.join(d, "include")) }
-abort "yeptris sources not found" unless src_root
+# Source root: YEPTRIS_SRC (CI / explicit), then sibling checkouts.
+src_roots = [ENV["YEPTRIS_SRC"]].compact
+src_roots << File.expand_path("../../../../yeptris/src", __dir__)
+src_roots << File.expand_path("../../../yeptris/src", __dir__)
+src_root = src_roots.find { |d| d && File.directory?(File.join(d, "include")) }
+abort "yeptris sources not found (set YEPTRIS_SRC)" unless src_root
 
 $INCFLAGS << " -I#{src_root}/include -I#{src_root}/yeptris"
 %w[build-validate/generated build/generated].each do |g|

data/ext/yeptris_native/json_ruby.c

@@ -1,14 +1,14 @@
 /* json_ruby.c — fused RFC 8259 → Ruby VALUE (TODO.restructure/22). */
 #include <ruby.h>
 #include <ruby/encoding.h>
+#include <ruby/intern.h>
 #include <stdlib.h>
 #include <string.h>
-#include "parse/numbers.h"
 #include "parse/scalars.h"
 #include "scan/json.h"
 
 #define YEP_JR_MAX 1000
-#define YEP_KC 256
+#define YEP_KC 1024
 
 typedef struct {
     uint64_t h;
@@ -86,40 +86,37 @@ static VALUE jr_str(jr* j, int as_key) {
         sp = j->p + start + 1;
         sl = (long)(close - start - 1);
     }
-    if (as_key || sl <= 2) return jr_cached(j, sp, sl);
+    if (as_key || sl <= 24) return jr_cached(j, sp, sl);
     return rb_enc_str_new(sp, sl, j->enc);
 }
 
 static VALUE jr_num(jr* j) {
     size_t start = j->i;
-    if (!yep_json_number(j->p, j->len, &j->i)) { j->err = -2; return Qnil; }
-    const char* s = j->p + start;
-    uint32_t n = (uint32_t)(j->i - start);
-    int is_float = 0, neg = 0;
-    uint32_t k = 0;
-    if (s[0] == '-') { neg = 1; k = 1; }
-    for (; k < n; k++) {
-        char c = s[k];
-        if (c == '.' || c == 'e' || c == 'E') { is_float = 1; break; }
+    int shape = 0;
+    int64_t iv = 0;
+    double dv = 0.0;
+    /* the fused kernel (scan/json.h): ONE grammar walk, values out */
+    if (!yep_json_number_scan(j->p, j->len, &j->i, &shape, &iv, &dv)) {
+        j->err = -2;
+        return Qnil;
     }
-    if (!is_float && n - (uint32_t)neg <= 18) {
-        int64_t v = 0;
-        for (k = (uint32_t)neg; k < n; k++) v = v * 10 + (s[k] - '0');
-        if (neg) v = -v;
-        return LL2NUM(v);
+    if (shape == 0) {
+        return LL2NUM(iv);
     }
-    if (is_float) {
-        double d = 0.0;
-        if (yep_num_f64(s, n, &d) != 0) { j->err = -2; return Qnil; }
-        return DBL2NUM(d);
+    if (shape == 1) {
+        return DBL2NUM(dv);
     }
-    int64_t v = 0;
-    if (yep_num_i64(s, n, &v) != 0) {
-        double d = 0.0;
-        if (yep_num_f64(s, n, &d) != 0) { j->err = -2; return Qnil; }
-        return DBL2NUM(d);
+    /* integer text beyond int64: exact Bignum from the validated
+     * span (JSON.parse's behavior). Absurd lengths degrade to the
+     * approximate double. */
+    size_t n = j->i - start;
+    if (n < 512) {
+        char buf[512];
+        memcpy(buf, j->p + start, n);
+        buf[n] = '\0';
+        return rb_cstr_to_inum(buf, 10, TRUE);
     }
-    return LL2NUM(v);
+    return DBL2NUM(dv);
 }
 
 static VALUE jr_object(jr* j) {
@@ -127,23 +124,44 @@ static VALUE jr_object(jr* j) {
     VALUE h = rb_hash_new_capa(8);
     jr_ws(j);
     if (j->i < j->len && j->p[j->i] == '}') { j->i++; j->depth--; return h; }
+    /* pairs collect into a flat buffer and land in ONE bulk insert:
+     * rb_hash_bulk_insert skips the per-pair method dispatch the
+     * aset loop pays (TODO.restructure/26's margin work) */
+    VALUE pairs[64];
+    VALUE* pv = pairs;
+    size_t pcap = 64, pn = 0, heap_cap = 0;
     for (;;) {
         jr_ws(j);
-        if (j->i >= j->len || j->p[j->i] != '"') { j->err = -2; return Qnil; }
+        if (j->i >= j->len || j->p[j->i] != '"') { j->err = -2; goto out; }
         VALUE key = jr_str(j, 1);
-        if (j->err) return Qnil;
+        if (j->err) goto out;
         jr_ws(j);
-        if (j->i >= j->len || j->p[j->i] != ':') { j->err = -2; return Qnil; }
+        if (j->i >= j->len || j->p[j->i] != ':') { j->err = -2; goto out; }
         j->i++;
         VALUE val = jr_value(j);
-        if (j->err) return Qnil;
-        rb_hash_aset(h, key, val);
+        if (j->err) goto out;
+        if (pn + 2 > pcap) {
+            size_t ncap = pcap * 2;
+            VALUE* nv = malloc(ncap * sizeof(VALUE));
+            if (!nv) { j->err = -1; goto out; }
+            memcpy(nv, pv, pn * sizeof(VALUE));
+            if (pv != pairs) { free(pv); heap_cap = 1; }
+            pv = nv; pcap = ncap;
+        }
+        pv[pn++] = key;
+        pv[pn++] = val;
         jr_ws(j);
-        if (j->i >= j->len) { j->err = -2; return Qnil; }
+        if (j->i >= j->len) { j->err = -2; goto out; }
         if (j->p[j->i] == ',') { j->i++; continue; }
-        if (j->p[j->i] == '}') { j->i++; j->depth--; return h; }
-        j->err = -2; return Qnil;
+        if (j->p[j->i] == '}') { j->i++; break; }
+        j->err = -2; goto out;
     }
+    rb_hash_bulk_insert((long)pn, (const VALUE*)pv, h);
+out:
+    if (pv != pairs) { free(pv); (void)heap_cap; }
+    if (j->err) return Qnil;
+    j->depth--;
+    return h;
 }
 
 static VALUE jr_array(jr* j) {

data/lib/yeptris/json.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Yeptris
+  # The STRICT JSON surface (TODO.restructure/31).
+  #
+  # `Yeptris::JSON.load` targets EXACT `JSON.parse` semantics — that
+  # is its parity target, spec-pinned (spec/json_parity_spec.rb).
+  # It is deliberately separate from Yeptris::YAML: the YAML surface
+  # keeps the Psych contract for every input (JSON-shaped included),
+  # so the two never drift into each other's semantics.
+  #
+  # Engines, fastest first (both exact — the parity spec runs against
+  # whichever is loaded):
+  # 1. the native materializer (fused C scan → VALUE; opt-in build)
+  # 2. the record drain + strict conversion walk (always available)
+  module JSON
+    class Error < ::Yeptris::Error; end
+    class ParseError < Error; end
+
+    module_function
+
+    def load(source)
+      source = ::Yeptris.read_input(source)
+      source = source.to_s
+      if defined?(::Yeptris::Native)
+        begin
+          return ::Yeptris::Native.load_json(source)
+        rescue ::Yeptris::ParseError => e
+          raise ParseError, e.message
+        end
+      end
+      strict_fallback(source)
+    end
+
+    # The always-available engine: the strict-JSON validator gates
+    # (parse_json raises on anything RFC 8259 rejects), then the
+    # value records convert WITHOUT the Psych quirk table — floats
+    # are always Floats, bools always bools ("1e3" is 1000.0 here
+    # and a String on the YAML surface; each surface is its own
+    # contract).
+    def strict_fallback(source)
+      begin
+        gate = ::Yeptris::Document.parse_json(source)
+      rescue ::Yeptris::ParseError => e
+        raise ParseError, e.message
+      end
+      begin
+        cols = ::Yeptris::FFI::ValueColumns.new
+        st = ::Yeptris::FFI.yeptris_value_drain_columns(
+          source, source.bytesize, ::Yeptris::FFI::SCHEMA_12_CORE, cols
+        )
+        raise ParseError, ::Yeptris::FFI.last_error_message if st != ::Yeptris::FFI::OK
+
+        begin
+          walk_strict(cols)
+        ensure
+          ::Yeptris::FFI.yeptris_value_free_columns(cols)
+        end
+      ensure
+        gate.free
+      end
+    end
+
+    # Placement mechanics mirror ValueML.walk_columns; the CONVERSION
+    # is the strict-JSON one (no ':sym' scan, no y/n quirk, no
+    # dot-required floats). Anchors/aliases/timestamps cannot occur
+    # in strict JSON — reaching them is an internal error.
+    def walk_strict(cols)
+      n = cols[:count]
+      kinds = cols[:kinds].read_bytes(n).unpack("C*")
+      ikeys = cols[:is_keys].read_bytes(n).unpack("C*")
+      bools = cols[:bools].read_bytes(n).unpack("C*")
+      offs = cols[:offs].read_bytes(n * 4).unpack("V*")
+      lens = cols[:lens].read_bytes(n * 4).unpack("V*")
+      pays = cols[:payloads].read_bytes(n * 8).unpack("q<*")
+      arena = cols[:arena_len].zero? ? +"" : cols[:arena].read_bytes(cols[:arena_len])
+      arena.force_encoding(Encoding::UTF_8)
+
+      docs = []
+      stack = []
+      pending_key = nil
+      i = 0
+      while i < n
+        case kinds[i]
+        when ValueML::DOC
+          docs.push(nil)
+        when ValueML::SEQ_OPEN
+          place(docs, stack, pending_key) { [] }
+          pending_key = nil
+        when ValueML::MAP_OPEN
+          place(docs, stack, pending_key) { {} }
+          pending_key = nil
+        when ValueML::CLOSE
+          stack.pop
+        when ValueML::V_STR
+          text = arena.byteslice(offs[i], lens[i])
+          if ikeys[i] == 1 && !stack.empty? && stack.last.is_a?(Hash)
+            pending_key = text
+          else
+            # Records carry int64 payloads: an integer-beyond-int64
+            # degrades to a PLAIN string (b==1). In strict JSON a
+            # plain (unquoted) scalar can ONLY be a number — every
+            # real string is quoted and arrives b==0 — so rebuild the
+            # exact Integer (JSON.parse parity, Bignum included).
+            if bools[i] == 1
+              place(docs, stack, pending_key) { Integer(text, 10) }
+            else
+              place(docs, stack, pending_key) { text }
+            end
+            pending_key = nil
+          end
+        when ValueML::V_INT
+          place(docs, stack, pending_key) { pays[i] }
+          pending_key = nil
+        when ValueML::V_FLOAT
+          place(docs, stack, pending_key) { [pays[i]].pack("q<").unpack1("E") }
+          pending_key = nil
+        when ValueML::V_BOOL
+          place(docs, stack, pending_key) { bools[i] == 1 }
+          pending_key = nil
+        when ValueML::V_NULL
+          place(docs, stack, pending_key) { nil }
+          pending_key = nil
+        else
+          raise Error, "internal: impossible record #{kinds[i]} in strict JSON"
+        end
+        i += 1
+      end
+      docs.empty? ? nil : docs.first
+    end
+
+    def place(docs, stack, key)
+      v = yield
+      if stack.empty?
+        docs[-1] = v
+      elsif key
+        stack.last[key] = v
+      else
+        stack.last.push(v)
+      end
+      stack.push(v) if v.is_a?(Array) || v.is_a?(Hash)
+      v
+    end
+  end
+end

data/lib/yeptris/yaml.rb

@@ -10,16 +10,14 @@ module Yeptris
     # Loads the FIRST document of a YAML stream as native Ruby objects.
     # schema: :compat_11 selects Psych/libyaml implicit typing
     # (yes/no, 0o/octal, sexagesimal); :core_12 (default) is YAML 1.2.
+    #
+    # This surface keeps the Psych contract for EVERY input — including
+    # JSON-shaped ones (`{"a": [1,]}` is legal flow YAML; `"1e3"` is a
+    # Psych String). Strict RFC 8259 semantics live on Yeptris::JSON
+    # (TODO.restructure/31): defaults follow proof, not benchmarks.
     def load(yaml, schema: :compat_11)
       yaml = Yeptris.read_input(yaml)
       yaml = yaml.to_s
-      # The native C materializer fuses strict-JSON scan→VALUE in one
-      # pass (beats JSON.parse on the 152 KB corpus). YAML inputs keep
-      # the FFI ladder so timestamps, aliases, and Psych's scalar
-      # quirks all resolve through the same path Psych.load uses.
-      if defined?(Yeptris::Native) && native_json?(yaml)
-        return Yeptris::Native.load_json(yaml)
-      end
       docs = _drain_all(yaml, schema)
       docs.empty? ? nil : docs.first
     end
@@ -31,22 +29,6 @@ module Yeptris
       _drain_all(yaml, schema)
     end
 
-    # Strict-JSON sniff: first non-space is { or [ — the fused native
-    # path beats JSON.parse on this shape (TODO.restructure/22).
-    def native_json?(bytes)
-      i = 0
-      len = bytes.bytesize
-      while i < len
-        c = bytes.getbyte(i)
-        return true if c == 0x7b || c == 0x5b # { or [
-        return false unless c == 0x20 || c == 0x09 || c == 0x0a || c == 0x0d
-
-        i += 1
-      end
-      false
-    end
-    private_class_method :native_json?
-
     # The Marshal fast path when the loaded libyeptris has it (>= 0.1.11
     # era builds), falling back to the columnar drain and finally the
     # record drain — one code path, the fastest the library offers.

data/lib/yeptris.rb

@@ -3,7 +3,7 @@
 module Yeptris
   # The gem's version lives in the parent namespace's file — the last
   # internal require (yeptris/version) retired with it.
-  VERSION = "0.1.12.0".freeze
+  VERSION = "0.1.13.0".freeze
   # The error hierarchy lives in THIS file (the parent namespace's
   # own file): nested constants do not trigger a parent-constant
   # autoload, and the law forbids internal requires — defining the
@@ -39,6 +39,7 @@ module Yeptris
   autoload :Document, "yeptris/document"
   autoload :Node, "yeptris/node"
   autoload :YAML, "yeptris/yaml"
+  autoload :JSON, "yeptris/json"
   autoload :Materializer, "yeptris/materializer"
   autoload :ValueML, "yeptris/valueml"
   autoload :Psych, "yeptris/psych"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yeptris
 version: !ruby/object:Gem::Version
-  version: 0.1.12.0
+  version: 0.1.13.0
 platform: ruby
 authors:
 - Ribose Inc.
@@ -40,6 +40,7 @@ files:
 - lib/yeptris.rb
 - lib/yeptris/document.rb
 - lib/yeptris/ffi.rb
+- lib/yeptris/json.rb
 - lib/yeptris/materializer.rb
 - lib/yeptris/node.rb
 - lib/yeptris/psych.rb