yeptris 0.6.8.1 → 0.6.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b263a55abf16d57caf2c85268f108dc87059f9188c21e1b2a622129a6a8056f9
-  data.tar.gz: dcd7ae19f1d52c9615d1d4dd169a632483e8bc40f6303f0f4591aff8b1684c30
+  metadata.gz: e582f07e09108f29273584fd4e7cb2387bc9410fda6d11a2f0209c40211d8a1c
+  data.tar.gz: 717cca7da0595ecd5ed4d356deabeca49343ce9d81cb40d13d5d2056595d0881
 SHA512:
-  metadata.gz: 999a6efa21ae21f9defc24546fa66e083ae9cf95dde6e6d93266c5ad75542b7c1a2923d19bda028124c1f825e447ab4062faf44ba31d6bf1199447e51492d582
-  data.tar.gz: 13afb6d81ca5795279dcebd790b4fae32d3e41abff68faacc7510d20c6557d0d358fb1bd217feee8d95b7aeb21b7b36cfc6f0d7f21e31a5e92e6e97f1f5c1292
+  metadata.gz: 552e9e7a9a024ebfb948717acbd15d845844159bd9b66fe4b2db29b5ae611e9cc09584aa1b66fc2a38c813532b4ba779c750b0c5e10cb0ecd81fe960afb5e3e8
+  data.tar.gz: f2a9fc0879fae9b0c498203552d48ac7574807eba66187e28e551efaff472f74dc3881fb4a43d552494a59b75f1af2298b6b0b2fa3167d5aabc8d5231a4ef4f8

data/lib/yeptris/ffi.rb

@@ -66,10 +66,16 @@ module Yeptris
     # the caller's string (no arena copy, no second validating parse).
     # v2 records: numbers are spans at parse; yeptris_tape_convert
     # materializes them in one bulk call.
+    # v3 (libyeptris 0.6.9): the interleaved records (recs) are the
+    # primary storage on the lenient route; the strict route keeps the
+    # columns eager. The layout MUST mirror yeptris_json_tape — a stale
+    # layout makes C write past the FFI buffer (a silent heap overflow
+    # that only some allocators catch; the 0.6.9.1 windows crash).
     class JsonTape < ::FFI::Struct
       layout :count, :size_t, :kinds, :pointer, :offs, :pointer,
-             :lens, :pointer, :int_min, :int64, :_src, :pointer, :_srclen, :size_t,
-             :_block, :pointer
+             :lens, :pointer, :recs, :pointer, :_cols_ready, :int,
+             :_rec_primary, :int, :int_min, :int64, :_src, :pointer,
+             :_srclen, :size_t, :_block, :pointer
     end
 
     attach_function :yeptris_parse_json_tape, %i[pointer size_t pointer], :int

data/lib/yeptris/json.rb

@@ -24,6 +24,7 @@ module Yeptris
     # behavior — strictness follows it (issue #37, found by canon's
     # CI where json 3.0.0 resolved while the dev box had 2.x).
     require "json"
+    require "date" # place_obj's `when Date` — psych loads it elsewhere, JSON.dump must not depend on that
     STRICT_DUPLICATE_KEYS = Gem::Version.new(::JSON::VERSION) >= Gem::Version.new("3")
 
     module_function

data/lib/yeptris/psych/drop_in.rb

@@ -21,3 +21,22 @@ end
 # rebind is this file's whole purpose
 Object.class_eval { remove_const(:Psych) } if defined?(::Psych) && !::Psych.equal?(Yeptris::Psych)
 ::Psych = Yeptris::Psych
+
+# yaml-first boot order: ::YAML still references the ORIGINAL stdlib
+# module object, whose singleton methods route through stdlib's parse
+# stream — the UTF-8 tagging (and every other normalization) never
+# runs (#135's spec/sdo discriminant). Delegate the original's public
+# singleton surface to the Yeptris face so BOTH constants behave
+# identically regardless of which module object a caller holds.
+# ORIGINAL exists only when stdlib psych was already loaded (the
+# drop-in-before-psych order needs no delegation — nothing references
+# the original yet).
+if ::Yeptris::Psych.const_defined?(:ORIGINAL, false)
+  ::Psych::ORIGINAL.singleton_class.class_eval do
+    ::Psych::ORIGINAL.singleton_methods(false).each do |m|
+      define_method(m) do |*args, **kwargs, &block|
+        ::Yeptris::Psych.public_send(m, *args, **kwargs, &block)
+      end
+    end
+  end
+end

data/lib/yeptris.rb

@@ -3,7 +3,7 @@
 module Yeptris
   # The gem's version lives in the parent namespace's file — the last
   # internal require (yeptris/version) retired with it.
-  VERSION = "0.6.8.1".freeze
+  VERSION = "0.6.9.1".freeze
   # The error hierarchy lives in THIS file (the parent namespace's
   # own file): nested constants do not trigger a parent-constant
   # autoload, and the law forbids internal requires — defining the

data/vendor/libyeptris/CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 project(yeptris
-    VERSION 0.6.8
+    VERSION 0.6.9
     DESCRIPTION "Ultra-fast YAML 1.2 parser, emitter and streamer in C"
     LANGUAGES C CXX
 )

data/vendor/libyeptris/src/include/yeptris/tape.h

@@ -46,15 +46,32 @@ enum {
                         spans; simdjson's deferred contract) */
 };
 
+/* The interleaved record (TODO.max-perf/07): one 8-byte store per
+ * token instead of three column stores. Layout: off:u32 | len:u23 |
+ * kind:u8 — a span longer than 0xFEFFFF bytes carries len == 0xFFFFFF
+ * with the true length stored as a CONT-terminated extension record
+ * (kind YEP_T_CONT; unreachable for <16 MiB tokens). The columns
+ * stay the compatibility ABI: yeptris_tape_columns materializes them
+ * lazily from the records on first touch. */
+#define YEP_T_CONT 8 /* record-length extension (not a token kind) */
+typedef uint64_t yeptris_tape_rec;
+
 typedef struct yeptris_json_tape {
     size_t count;
-    uint8_t* kinds;   /* yeptris tape kind per record */
-    uint32_t* offs;   /* span starts (STR/INT/FLOAT); container links */
-    uint32_t* lens;   /* span lengths (STR/INT/FLOAT) */
-    int64_t int_min;  /* set by yeptris_tape_convert when an INT span
-                         exceeds int64 (materialize-time discovery) */
-    const void* _src; /* the parsed buffer (spans borrow it; it must
-                         outlive the tape — same contract as the spans) */
+    uint8_t* kinds;         /* compat columns — materialized lazily (see
+                               yeptris_tape_columns) when the interleaved
+                               records are the primary storage */
+    uint32_t* offs;         /* span starts (STR/INT/FLOAT); container links */
+    uint32_t* lens;         /* span lengths (STR/INT/FLOAT) */
+    yeptris_tape_rec* recs; /* primary storage (may be NULL on legacy
+                               column-built tapes: the strict route) */
+    int _cols_ready;        /* columns materialized from recs already */
+    int _rec_primary;       /* the lenient route: recs carry the data and
+                               the columns are lazy */
+    int64_t int_min;        /* set by yeptris_tape_convert when an INT span
+                               exceeds int64 (materialize-time discovery) */
+    const void* _src;       /* the parsed buffer (spans borrow it; it must
+                               outlive the tape — same contract as the spans) */
     size_t _srclen;
     void* _block; /* the carved allocation base */
 } yeptris_json_tape;
@@ -89,6 +106,13 @@ YEPTRIS_API void yeptris_tape_free(yeptris_json_tape* tape);
  * number records converted, or SIZE_MAX if a span does not re-scan
  * as a number (impossible for a tape this library produced).
  * Sets t->int_min when an INT span exceeds int64. */
+/* Materializes the kinds/offs/lens columns from the interleaved
+ * records (no-op when the tape was built column-primary or the
+ * columns are already materialized). Consumers reading the column
+ * pointers directly (the FFI binding) must call this once first.
+ * Returns 0 on success, nonzero on allocation failure. */
+YEPTRIS_API int yeptris_tape_columns(yeptris_json_tape* t);
+
 YEPTRIS_API size_t yeptris_tape_convert(yeptris_json_tape* t, size_t from, size_t to,
                                         int64_t* ivals, double* dvals);
 

data/vendor/libyeptris/src/yeptris/plan.c

@@ -415,6 +415,9 @@ static yeptris_plan_result* plan_result_carve(const yeptris_plan* plan, size_t r
 
 YEPTRIS_API yeptris_plan_result*
 yeptris_tape_plan_walk(const yeptris_json_tape* tape, const yeptris_plan* plan, YeptrisStatus* st) {
+    /* item 07: the lenient tape's columns are lazy — materialize (a
+     * cache write through a const view; idempotent) */
+    yeptris_tape_columns((yeptris_json_tape*)tape);
     if (st != NULL) {
         *st = YEPTRIS_OK;
     }

data/vendor/libyeptris/src/yeptris/tape.c

@@ -59,7 +59,8 @@ static int rec_put(tape_ctx* c, uint8_t kind, uint32_t off, uint32_t len) {
 static YeptrisStatus tape_carve(yeptris_json_tape* t, size_t len) {
     size_t cap = len + 2;
     size_t off_o = (cap + 15) & ~(size_t)15;
-    char* block = yep_alloc(yep_system_allocator(), off_o + 2 * cap * sizeof(uint32_t));
+    char* block = yep_alloc(yep_system_allocator(),
+                            off_o + 2 * cap * sizeof(uint32_t) + cap * sizeof(yeptris_tape_rec));
     if (block == NULL) {
         return YEPTRIS_ERROR_MEMORY;
     }
@@ -67,11 +68,31 @@ static YeptrisStatus tape_carve(yeptris_json_tape* t, size_t len) {
     t->kinds = (uint8_t*)block;
     t->offs = (uint32_t*)(void*)(block + off_o);
     t->lens = t->offs + cap;
+    t->recs = (yeptris_tape_rec*)(void*)(t->lens + cap);
     t->count = 0;
     t->int_min = 0;
+    t->_rec_primary = 0;
+    t->_cols_ready = 0;
     return YEPTRIS_OK;
 }
 
+/* The lazy column materialization (TODO.max-perf/07): records are the
+ * primary storage on the lenient route; the column ABI materializes
+ * from them on first touch. */
+YEPTRIS_API int yeptris_tape_columns(yeptris_json_tape* t) {
+    if (t == NULL || !t->_rec_primary || t->recs == NULL || t->_cols_ready) {
+        return 0; /* column-primary (the strict route) or ready */
+    }
+    for (size_t i = 0; i < t->count; i++) {
+        yeptris_tape_rec r = t->recs[i];
+        t->lens[i] = (uint32_t)((r >> 8) & 0xFFFFFFu);
+        t->offs[i] = (uint32_t)(r >> 32);
+        t->kinds[i] = (uint8_t)(r & 0xFFu);
+    }
+    t->_cols_ready = 1;
+    return 0;
+}
+
 /* Scalar roots: the walk needs an opener, so a bare root value
  * converts through the same kernels. */
 static int tape_put_root_scalar(tape_ctx* c, const char* p, size_t len, size_t at) {
@@ -585,20 +606,18 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
     if (tape_carve(t, len) != YEPTRIS_OK) {
         return YEPTRIS_ERROR_MEMORY;
     }
-    uint8_t* kinds = t->kinds;
-    uint32_t* offs = t->offs;
-    uint32_t* lens = t->lens;
+    /* the interleaved records are the primary storage (item 07); the
+     * columns materialize lazily via yeptris_tape_columns */
+    t->_rec_primary = 1;
+    yeptris_tape_rec* recs = t->recs;
     uint32_t open_at[YEP_JSON_WALK_DEPTH];
     uint8_t kind[YEP_JSON_WALK_DEPTH];
 
-    kinds[0] = YEP_T_DOC;
-    offs[0] = 0;
-    lens[0] = 0;
+    recs[0] = ((uint64_t)0 << 32) | ((uint64_t)0 << 8) | YEP_T_DOC;
 
     uint8_t top_kind = p[open] == '[' ? 0 : 1;
-    kinds[1] = top_kind ? YEP_T_MAP_OPEN : YEP_T_SEQ_OPEN;
-    offs[1] = 0;
-    lens[1] = 0;
+    recs[1] =
+        ((uint64_t)0 << 32) | ((uint64_t)0 << 8) | (top_kind ? YEP_T_MAP_OPEN : YEP_T_SEQ_OPEN);
     uint32_t top_open = 1;
     size_t count = 2;
     uint8_t top_expect = top_kind ? JW_KEY_OR_CLOSE : JW_VALUE_OR_CLOSE;
@@ -656,9 +675,8 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
                 uint64_t c0 = (w - 0x2020202020202020ull) & ~w & 0x8080808080808080ull;
                 if (qm != 0 && ((bm | c0) & (qm - 1)) == 0) {
                     close = j + (size_t)yep_ctz64(qm) / 8;
-                    kinds[count] = YEP_T_STR;
-                    offs[count] = (uint32_t)j;
-                    lens[count] = (uint32_t)(close - j);
+                    recs[count] = ((uint64_t)((uint32_t)j) << 32) |
+                                  ((uint64_t)((uint32_t)(close - j)) << 8) | (uint64_t)(YEP_T_STR);
                     count++;
                     i = close + 1;
                     goto lstr_done;
@@ -668,9 +686,8 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
             if (!yep_json_string(p, len, &i, &close, &esc)) {
                 goto lreject;
             }
-            kinds[count] = YEP_T_STR;
-            offs[count] = (uint32_t)(at + 1);
-            lens[count] = (uint32_t)(close - at - 1);
+            recs[count] = ((uint64_t)((uint32_t)(at + 1)) << 32) |
+                          ((uint64_t)((uint32_t)(close - at - 1)) << 8) | (uint64_t)(YEP_T_STR);
             count++;
         lstr_done:
             if (key_slot) {
@@ -698,9 +715,8 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
                 }
                 break;
             }
-            kinds[count] = YEP_T_NUM;
-            offs[count] = (uint32_t)at;
-            lens[count] = (uint32_t)(i - at);
+            recs[count] = ((uint64_t)((uint32_t)at) << 32) | ((uint64_t)((uint32_t)(i - at)) << 8) |
+                          (uint64_t)(YEP_T_NUM);
             count++;
         lcomma:
             if (i < len && p[i] == ',') {
@@ -719,10 +735,11 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
                  top_expect != JW_COMMA_OR_CLOSE)) {
                 goto lreject;
             }
-            kinds[count] = YEP_T_CLOSE;
-            offs[count] = top_open;
-            lens[count] = 0;
-            offs[top_open] = (uint32_t)count;
+            recs[count] = ((uint64_t)top_open << 32) | ((uint64_t)0 << 8) | YEP_T_CLOSE;
+            /* back-patch the opener's count link: keep the original
+             * off/len, set the link into the record's off word */
+            recs[top_open] = (recs[top_open] & ~(uint64_t)0xFFFFFFFF00000000u) |
+                             ((uint64_t)(uint32_t)count << 32);
             count++;
             i = at + 1;
             depth--;
@@ -745,9 +762,8 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
             kind[depth - 1] = top_kind;
             open_at[depth - 1] = top_open;
             top_kind = c == '[' ? 0 : 1;
-            kinds[count] = c == '[' ? YEP_T_SEQ_OPEN : YEP_T_MAP_OPEN;
-            offs[count] = 0;
-            lens[count] = 0;
+            recs[count] = ((uint64_t)0 << 32) | ((uint64_t)0 << 8) |
+                          (uint64_t)(c == '[' ? YEP_T_SEQ_OPEN : YEP_T_MAP_OPEN);
             top_open = (uint32_t)count;
             count++;
             top_expect = top_kind ? JW_KEY_OR_CLOSE : JW_VALUE_OR_CLOSE;
@@ -778,9 +794,8 @@ static YeptrisStatus tape_walk_lnt_fused(const char* p, size_t len, size_t open,
                     goto lreject;
                 }
             }
-            kinds[count] = c == 'n' ? YEP_T_NULL : (c == 't' ? YEP_T_TRUE : YEP_T_FALSE);
-            offs[count] = (uint32_t)at;
-            lens[count] = (uint32_t)wl;
+            recs[count] = ((uint64_t)((uint32_t)at) << 32) | ((uint64_t)((uint32_t)wl) << 8) |
+                          (uint64_t)(c == 'n' ? YEP_T_NULL : (c == 't' ? YEP_T_TRUE : YEP_T_FALSE));
             count++;
             i = at + wl;
             goto lcomma;
@@ -1244,6 +1259,7 @@ YEPTRIS_API YeptrisStatus yeptris_parse_json_tape(const char* source, size_t len
 
 YEPTRIS_API size_t yeptris_tape_convert(yeptris_json_tape* t, size_t from, size_t to,
                                         int64_t* ivals, double* dvals) {
+    yeptris_tape_columns(t);
     if (t == NULL || t->_src == NULL || from > to || to > t->count) {
         return SIZE_MAX;
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yeptris
 version: !ruby/object:Gem::Version
-  version: 0.6.8.1
+  version: 0.6.9.1
 platform: ruby
 authors:
 - Ribose Inc.