teptris 0.2.51-aarch64-linux → 0.2.53-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c06b1c6b4465d52d8bac5ee2f88358a37c78ddac94a490be41cffab7928feed
-  data.tar.gz: 33806d8766934f6c2e9a24045dc59a5949dcec7265676df819bd7b11db78da57
+  metadata.gz: 168a5dbc5e797d21c3d6798cd78b90d4b8a704b1f9423413904c71b2559c31fb
+  data.tar.gz: 2e6f06fcfc14d22db8c821402462d428a09dd30c4c63e4ac5d071173415084a8
 SHA512:
-  metadata.gz: 6a92ba34fa093303d656d442ea4541e7a1d9b2cd2a34211d414110858407904f5e67198281eeeaa4c8e627d4bb5dc553bc0bffbab0a3745bd0b42324a630eb2b
-  data.tar.gz: 91d21740161a5872a7fcee8f563ee8eff506528f1a6b10a2e17c18e9cb95637ce82cd7041b9b8f773573c3ae57e667df8b69bfe649aa548e1ea5ed2cd9cd331d
+  metadata.gz: 050ed9059cefb4cc60b37194b353e9a385335aa22e99395ed993833967efa7d466e0e3c0de2d9764ee08b528f56cf6e7e622aa6f3d0f0c5503931a395746ea55
+  data.tar.gz: 4c24b94cbe121c7413367f42afd48413e2acce70dd47328614d9f7da94735d91b74cfdbc30d44404e8b46f89519fa75a2e7ad64fd30f501c07d3798e10a5fd32

data/ext/teptris_ext/materialize.c

@@ -1,4 +1,5 @@
 #include <ruby.h>
+#include <ruby/thread.h>
 #include <ruby/encoding.h>
 #include "teptris/teptris.h"
 #include "teptris/plan.h"
@@ -237,20 +238,25 @@ static VALUE obj_from_node(const teptris_node *n, unsigned flags,
     }}
 }
 
-static teptris_document *parse_or_raise(VALUE str) {
-    teptris_document *doc = NULL;
-    teptris_status st = teptris_parse(RSTRING_PTR(str), (size_t)RSTRING_LEN(str), NULL, &doc);
-    if (st != TEPTRIS_OK) {
-        const teptris_error *e = teptris_document_error(doc);
-        VALUE ex = rb_exc_new(eParseError, e->message, (long)strlen(e->message));
-        rb_iv_set(ex, "@line", SIZET2NUM(e->line));
-        rb_iv_set(ex, "@column", SIZET2NUM(e->column));
-        teptris_document_free(doc);
-        rb_exc_raise(ex);
-    }
-    return doc;
+/* the parse is pure C over the buffer with no VM access: run it
+ * GVL-free so other threads progress during long parses (and parallel
+ * batch loads overlap). The string is locked (pinned) across the
+ * release so a concurrent compacting GC cannot relocate the buffer. */
+struct gvl_parse_arg {
+    const char *data;
+    size_t len;
+    teptris_document **doc;
+    teptris_status st;
+};
+
+static void *parse_without_gvl(void *p)
+{
+    struct gvl_parse_arg *a = (struct gvl_parse_arg *)p;
+    a->st = teptris_parse(a->data, a->len, NULL, a->doc);
+    return NULL;
 }
 
+
 static VALUE ext_load(int argc, VALUE *argv, VALUE self) {
     VALUE str, opts;
     rb_scan_args(argc, argv, "11", &str, &opts);
@@ -264,11 +270,29 @@ static VALUE ext_load(int argc, VALUE *argv, VALUE self) {
         if (RTEST(ft)) flags |= FMT_FORBID_TIME;
         if (RTEST(fd)) flags |= FMT_FORBID_DATE;
     }
-    teptris_document *doc = parse_or_raise(str);
+    /* the parse is pure C over a private copy, GVL-free; the copy
+     * must outlive materialization - the document's strings are
+     * zero-copy views into it (a real UAF on glibc when freed early) */
+    long len = RSTRING_LEN(str);
+    char *cbuf = ALLOC_N(char, (size_t)len + 1);
+    memcpy(cbuf, RSTRING_PTR(str), (size_t)len);
+    teptris_document *doc = NULL;
+    struct gvl_parse_arg arg = {cbuf, (size_t)len, &doc, TEPTRIS_OK};
+    rb_thread_call_without_gvl(parse_without_gvl, &arg, RUBY_UBF_IO, NULL);
+    if (arg.st != TEPTRIS_OK) {
+        const teptris_error *e = teptris_document_error(doc);
+        VALUE ex = rb_exc_new(eParseError, e->message, (long)strlen(e->message));
+        rb_iv_set(ex, "@line", SIZET2NUM(e->line));
+        rb_iv_set(ex, "@column", SIZET2NUM(e->column));
+        xfree(cbuf);
+        teptris_document_free(doc);
+        rb_exc_raise(ex);
+    }
     keycache kc;
     kc_init(&kc);
     VALUE out = obj_from_node(teptris_document_root(doc), flags, &kc);
     kc_free(&kc);
+    xfree(cbuf);
     teptris_document_free(doc);
     return out;
 }
@@ -713,7 +737,8 @@ static VALUE cLazyOwner, cLazyValue;
 
 typedef struct {
     teptris_document *doc;
-    VALUE input; /* the borrowed views' buffer */
+    VALUE input; /* single-doc path: owned frozen copy (GC-marked) */
+    char *buf;   /* batch path: owned input copy the doc's views use */
 } lazy_owner;
 
 typedef struct {
@@ -724,12 +749,18 @@ typedef struct {
 static void lazy_owner_free(void *p) {
     lazy_owner *o = (lazy_owner *)p;
     if (o->doc != NULL) teptris_document_free(o->doc);
+    if (o->buf != NULL) xfree(o->buf);
     xfree(o);
 }
 static size_t lazy_owner_size(const void *p) { return sizeof(lazy_owner); }
+static void lazy_owner_mark(void *p)
+{
+    lazy_owner *o = (lazy_owner *)p;
+    if (o->input) rb_gc_mark(o->input); /* roots the single-path copy */
+}
 static const rb_data_type_t lazy_owner_type = {
     "TeptrisExt/LazyOwner",
-    {NULL, lazy_owner_free, lazy_owner_size, NULL, 0},
+    {lazy_owner_mark, lazy_owner_free, lazy_owner_size, NULL, 0},
     NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY};
 
 static void lazy_value_mark(void *p) {
@@ -764,7 +795,7 @@ static VALUE ext_load_lazy(VALUE self, VALUE toml) {
     VALUE owner = TypedData_Make_Struct(cLazyOwner, lazy_owner,
                                         &lazy_owner_type, o);
     o->doc = NULL;
-    o->input = rb_str_new_frozen(toml); /* borrowed views: own a copy */
+    o->input = rb_str_new_frozen(toml); /* owned frozen copy */
     teptris_status st = teptris_parse(RSTRING_PTR(o->input),
                                       (size_t)RSTRING_LEN(o->input),
                                       NULL, &o->doc);
@@ -945,25 +976,53 @@ static unsigned flags_from_opts(VALUE opts) {
 /* Parse-only batch (returns an Array of Lazy wrappers, one per doc):
  * eager parse, deferred materialization. Same per-doc error semantics
  * as ext_load_lazy. */
+struct batch_parse_arg {
+    const char **data;
+    size_t *lens;
+    size_t n;
+    teptris_document **docs;
+    teptris_status *statuses;
+    teptris_status batch;
+};
+
+static void *batch_parse_without_gvl(void *p)
+{
+    struct batch_parse_arg *a = (struct batch_parse_arg *)p;
+    a->batch = teptris_parse_batch(a->data, a->lens, a->n, NULL,
+                                   a->docs, a->statuses);
+    return NULL;
+}
+
 static VALUE ext_load_lazy_batch(VALUE self, VALUE strings) {
     Check_Type(strings, T_ARRAY);
     long n = RARRAY_LEN(strings);
     if (n == 0) return rb_ary_new();
     batch_scratch *s = batch_scratch_alloc(n);
     if (s == NULL) rb_raise(eError, "out of memory batching %ld documents", n);
+    /* the batch parse is pure C: pin every input (one tmp lock per
+     * string - repeated objects dup; frozen strings cannot take the
+     * lock, yet compaction can move them) and release the GVL for the
+     * whole parse */
+    /* the batch parse is pure C over private copies: a Ruby-string
+     * buffer can move under a concurrent compacting GC, and locktmp
+     * does not pin on every platform - copies are C memory, immovable
+     * by construction. Each LazyOwner owns its copy (freed with the
+     * owner): the lazy nodes view these buffers until materialized. */
+    char **bufs = ALLOC_N(char *, n);
     for (long i = 0; i < n; i++) {
         VALUE str = rb_ary_entry(strings, i);
         StringValue(str); /* raises TypeError if not coercible */
-        s->data[i] = RSTRING_PTR(str);
-        s->lens[i] = (size_t)RSTRING_LEN(str);
-    }
-    teptris_status batch =
-        teptris_parse_batch(s->data, s->lens, (size_t)n, NULL,
-                            s->docs, s->statuses);
-    if (batch != TEPTRIS_OK) {
-        batch_scratch_free(s);
-        rb_raise(eError, "batch parse failed (status %d)", (int)batch);
+        long len = RSTRING_LEN(str);
+        bufs[i] = ALLOC_N(char, (size_t)len + 1);
+        memcpy(bufs[i], RSTRING_PTR(str), (size_t)len);
+        s->data[i] = bufs[i];
+        s->lens[i] = (size_t)len;
     }
+    struct batch_parse_arg arg = {s->data, s->lens, (size_t)n,
+                                  s->docs, s->statuses, TEPTRIS_OK};
+    rb_thread_call_without_gvl(batch_parse_without_gvl, &arg,
+                               RUBY_UBF_IO, NULL);
+    teptris_status batch = arg.batch;
     long first_fail = -1;
     for (long i = 0; i < n; i++) {
         if (s->statuses[i] != TEPTRIS_OK && first_fail == -1) first_fail = i;
@@ -984,7 +1043,7 @@ static VALUE ext_load_lazy_batch(VALUE self, VALUE strings) {
         VALUE owner = TypedData_Make_Struct(cLazyOwner, lazy_owner,
                                             &lazy_owner_type, o);
         o->doc = s->docs[i];
-        o->input = rb_ary_entry(strings, i);
+        o->buf = bufs[i]; /* the owner owns this copy */
         rb_ary_push(out, lazy_wrap(owner, teptris_document_root(s->docs[i])));
     }
     batch_scratch_free(s);
@@ -995,6 +1054,94 @@ static VALUE ext_load_lazy_batch(VALUE self, VALUE strings) {
  * as ext_load. opts is parsed once; safe_load semantics apply uniformly.
  * The caller's input Array is read-only; its strings stay referenced
  * for the duration of the C call (no separate keeper Array). */
+/* slice-level GVL release: one thread parses K documents under a
+ * single release (no per-document GVL ping-pong), then materializes
+ * them under the GVL with one shared key cache (batch semantics).
+ * All inputs are pinned (dup'd when frozen) across the release. */
+struct slice_parse_arg {
+    long n;
+    const char **data;
+    size_t *lens;
+    teptris_document **docs;
+    teptris_status *stats;
+};
+
+static void *slice_parse_without_gvl(void *p)
+{
+    struct slice_parse_arg *a = (struct slice_parse_arg *)p;
+    for (long i = 0; i < a->n; i++) {
+        a->docs[i] = NULL;
+        a->stats[i] = teptris_parse(a->data[i], a->lens[i], NULL, &a->docs[i]);
+    }
+    return NULL;
+}
+
+static VALUE ext_load_slice(int argc, VALUE *argv, VALUE self) {
+    VALUE strs, opts;
+    rb_scan_args(argc, argv, "11", &strs, &opts);
+    Check_Type(strs, T_ARRAY);
+    long n = RARRAY_LEN(strs);
+    if (n == 0) return rb_ary_new();
+    unsigned flags = flags_from_opts(opts);
+
+    /* the GVL-free parse reads private copies: a Ruby-string buffer
+     * can move under a concurrent compacting GC, and locktmp does not
+     * pin on every platform (windows ruby 3.3 parsed garbage). Copies
+     * are C memory - immovable by construction. Freed after
+     * materialization (the docs view them until then). */
+    const char **data = ALLOC_N(const char *, n);
+    size_t *lens = ALLOC_N(size_t, n);
+    teptris_document **docs = ALLOC_N(teptris_document *, n);
+    teptris_status *stats = ALLOC_N(teptris_status, n);
+    char **bufs = ALLOC_N(char *, n);
+    long i;
+    for (i = 0; i < n; i++) {
+        VALUE str = rb_ary_entry(strs, i);
+        StringValue(str);
+        long len = RSTRING_LEN(str);
+        bufs[i] = ALLOC_N(char, (size_t)len + 1);
+        memcpy(bufs[i], RSTRING_PTR(str), (size_t)len);
+        data[i] = bufs[i];
+        lens[i] = (size_t)len;
+    }
+    struct slice_parse_arg arg = {n, data, lens, docs, stats};
+    rb_thread_call_without_gvl(slice_parse_without_gvl, &arg,
+                               RUBY_UBF_IO, NULL);
+
+    VALUE out = rb_ary_new_capa(n);
+    keycache kc;
+    kc_init(&kc);
+    for (i = 0; i < n; i++) {
+        if (stats[i] != TEPTRIS_OK && docs[i] == NULL) {
+            const char *msg = "parse failed (out of memory)";
+            VALUE ex = rb_exc_new(eParseError, msg, (long)strlen(msg));
+            kc_free(&kc);
+            xfree(data); xfree(lens); xfree(docs); xfree(stats);
+            rb_exc_raise(ex);
+        }
+        const teptris_error *e = teptris_document_error(docs[i]);
+        if (stats[i] != TEPTRIS_OK) {
+            VALUE ex = rb_exc_new(eParseError, e->message,
+                                  (long)strlen(e->message));
+            rb_iv_set(ex, "@line", SIZET2NUM(e->line));
+            rb_iv_set(ex, "@column", SIZET2NUM(e->column));
+            long j;
+            for (j = i; j < n; j++) teptris_document_free(docs[j]);
+            kc_free(&kc);
+            xfree(data); xfree(lens); xfree(docs); xfree(stats);
+            rb_exc_raise(ex);
+        }
+    }
+    for (i = 0; i < n; i++) {
+        rb_ary_store(out, i,
+                     obj_from_node(teptris_document_root(docs[i]), flags, &kc));
+        teptris_document_free(docs[i]);
+    }
+    kc_free(&kc);
+    xfree(data); xfree(lens); xfree(docs); xfree(stats);
+    return out;
+}
+
 static VALUE ext_load_batch(int argc, VALUE *argv, VALUE self) {
     VALUE strings, opts;
     rb_scan_args(argc, argv, "11", &strings, &opts);
@@ -1052,6 +1199,7 @@ void Init_teptris_ext(void) {
     rb_define_module_function(m, "load", ext_load, -1);
     rb_define_module_function(m, "load_lazy", ext_load_lazy, 1);
     rb_define_module_function(m, "load_batch", ext_load_batch, -1);
+    rb_define_module_function(m, "load_slice", ext_load_slice, -1);
     rb_define_module_function(m, "load_lazy_batch", ext_load_lazy_batch, 1);
     cLazyOwner = rb_define_class_under(m, "LazyOwner", rb_cObject);
     cLazyValue = rb_define_class_under(m, "LazyValue", rb_cObject);

data/lib/teptris/toml.rb

@@ -51,9 +51,53 @@ module Teptris
       # struct across the batch, return one eager Hash per document.
       # Same datetime contract as load. A failing document raises
       # Teptris::ParseError carrying the first failure's line/column.
-      def load_batch(tomls, safe_load: nil, datetime_policy: :native)
+      def load_batch(tomls, safe_load: nil, datetime_policy: :native,
+                    threads: 1)
         _validate_batch!(tomls, safe_load, datetime_policy)
-        TeptrisExt.load_batch(tomls, _safe_load_opts(safe_load, datetime_policy))
+        opts = _safe_load_opts(safe_load, datetime_policy)
+        return TeptrisExt.load_batch(tomls, opts) if threads <= 1
+
+        # threads > 1: slice across Ruby threads. The C parse releases
+        # the GVL, so parses overlap; materialization serializes on the
+        # GVL. Contiguous slices keep error order deterministic: the
+        # lowest-index failure raises after all threads join.
+        n = tomls.size
+        threads = [threads, n].min
+        return TeptrisExt.load_batch(tomls, opts) if threads <= 1
+        # load_slice releases the GVL once per slice (no per-document
+        # ping-pong) and shares one key cache across its documents.
+        # Repeated objects are dup'd once each HERE: a repeat landing in
+        # two slices would double-lock across threads (the slice-level
+        # guard cannot see across slices); frozen dup'ing is C-side.
+        seen = {}
+        tomls = tomls.map do |s|
+          next s.dup if seen[s.object_id]
+          seen[s.object_id] = true
+          s
+        end
+        per = (n.to_f / threads).ceil
+        results = Array.new(n)
+        first_error = nil
+        mutex = Mutex.new
+        threads.times.map do |ti|
+          Thread.new(ti) do |i|
+            base = i * per
+            slice = tomls[base, per] || []
+            begin
+              loaded = TeptrisExt.load_slice(slice, opts)
+              slice.each_with_index { |_, j| results[base + j] = loaded[j] }
+            rescue StandardError => e
+              mutex.synchronize do
+                first_error ||= [base, e]
+              end
+            end
+          end
+        end.each(&:join)
+        if first_error
+          _, err = first_error
+          raise err
+        end
+        results
       end
 
       # Lazy twin of load_batch: parse N inputs eagerly, hand back an

data/lib/teptris/version.rb

@@ -1,3 +1,3 @@
 module Teptris
-  VERSION = "0.2.51"
+  VERSION = "0.2.53"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: teptris
 version: !ruby/object:Gem::Version
-  version: 0.2.51
+  version: 0.2.53
 platform: aarch64-linux
 authors:
 - teptris contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-30 00:00:00.000000000 Z
+date: 2026-10-01 00:00:00.000000000 Z
 dependencies: []
 description: A native C extension over libteptris, a C99 TOML 1.1 parser/emitter.
   API shape mirrors the tomlib gem.