yeptris 0.6.19.1-arm-linux-eabihf → 0.6.20.2-arm-linux-eabihf

data/vendor/libyeptris/src/yeptris/dom/ytape.h

@@ -0,0 +1,77 @@
+/* ytape.h — the packed YAML record tape (#378, TODO.max-perf/12).
+ *
+ * The eager DOM builders' callback stream packed into 8-byte words:
+ * the common block ops carry spans only (the resolver call and the
+ * 48-byte node build defer to replay), the general event path packs
+ * into a props word plus span words. Replay calls the SAME builder
+ * functions with reconstructed views — resolution and placement
+ * semantics are the eager path's by construction (the differential
+ * gate pins tree identity across the corpora).
+ *
+ * Spans are input-relative when they fall inside [input, input+len]
+ * and fit 24 bits; anything else (pool content, oversized spans) is a
+ * generic (ptr, len) pair — the engine's finish pool owns that
+ * memory, its blocks are stable until the document frees it, and the
+ * lazy route keeps the pool on the document until replay copies (the
+ * builders copy exactly what must outlive the engine — dom_str_in).
+ *
+ * The recorder mirrors the eager builders' return-code discipline
+ * (depth caps answer 0 so the engine's event path owns the error;
+ * OOM answers <0) so error precedence survives the detour. */
+#ifndef YEP_YTAPE_H
+#define YEP_YTAPE_H
+
+#include <stddef.h>
+
+#include "parse/events.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct yep_pool yep_pool;
+
+typedef struct yep_ytape {
+    uint64_t* w;
+    uint32_t count;
+    uint32_t cap;
+    const char* input; /* the POST-BOM data pointer (span base) */
+    size_t input_len;
+    int depth;       /* the DOM's depth mirror (cap parity) */
+    int max_depth;   /* the engine's flow-walk limit, replayed */
+    int flow_strict; /* mirrors dom->flow_strict (0 everywhere today) */
+    int docs;        /* DOCUMENT_START count (the empty-stream check) */
+    int oom;
+    uint32_t flow_mark; /* record mark for the staged flow build */
+} yep_ytape;
+
+/* Record kinds — byte 7 of a word. */
+enum {
+    YTP_SPAN_IN = 0,      /* [off:32][len:24][0x00] */
+    YTP_EV_COMPACT = 1,   /* stream/document frames: [type:8][...][kind] */
+    YTP_EVENT = 2,        /* props + value + [tag] + [anchor] */
+    YTP_SCALAR = 3,       /* props + value + [tag] + [anchor]; is_item bit
+                           * routes the replay to the dash-item builder */
+    YTP_ITEM = 4,         /* borrowed dash item: [off:32][len:24][kind] */
+    YTP_PAIR = 5,         /* props + key + value + [anchor] + [anchor hi] */
+    YTP_OPEN = 6,         /* `key:`'s fresh map: [off:32][len:24][kind] */
+    YTP_FLOW = 7,         /* props + [open:32|len:32] + [tag] + [anchor] */
+    YTP_SPAN_POOL = 0xFF, /* [len:32][kind] + [ptr:64] */
+};
+
+/* Recorder lifecycle. Returns 0, or -1 when the word carve fails
+ * (caller answers YEPTRIS_ERROR_MEMORY — bounded-parse law). */
+int ytap_init(yep_ytape* t, const char* input, size_t len, int max_depth);
+void ytap_sink(yep_ytape* t, yep_sink* sink);
+void ytap_free(yep_ytape* t);
+
+/* Replay: builds the node tree through the eager builders. Returns 0,
+ * or -1 (a builder refused — allocation/depth; the lazy route treats
+ * it as the JSON tape route does). Sets d->input_base/len. */
+int dom_from_ytape(yep_dom* d, const yep_ytape* t);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* YEP_YTAPE_H */

data/vendor/libyeptris/src/yeptris/emit/writer.c

@@ -91,7 +91,12 @@ static void wr_put(yep_writer* w, const char* p, uint32_t n) {
         w->len += n;
         return;
     }
-    if (w->grow && w->len + n > w->cap) {
+    /* the +1 reserves the terminator slot: the serializers write the
+     * NUL at p[len] after the run, and a write landing exactly on cap
+     * (len + n == cap) put that NUL one byte past the buffer — the
+     * first byte of the next heap block's header (the #182 flake:
+     * corrupt deterministic, detected later, timing-varying) */
+    if (w->grow && w->len + n + 1 > w->cap) {
         wr_grow(w, n);
     }
     if (w->oom) {
@@ -114,7 +119,7 @@ static void wr_byte(yep_writer* w, char c) {
         w->len++;
         return;
     }
-    if (w->grow && w->len + 1 > w->cap) {
+    if (w->grow && w->len + 2 > w->cap) { /* +1 write, +1 terminator */
         wr_grow(w, 1);
     }
     if (w->oom) {

data/vendor/libyeptris/src/yeptris/parse.c

@@ -5,6 +5,7 @@
 #include "common/simd_text.h"
 #include "doc.h"
 #include "dom/dom.h"
+#include "dom/ytape.h"
 #include "encoding/encoding.h"
 #include "memory/allocator.h"
 #include "memory/pool.h"
@@ -33,7 +34,7 @@ YEPTRIS_API const char* yeptris_last_error(uint32_t* line, uint32_t* col) {
 }
 
 static YeptrisDocument parse_impl(const char* buf, size_t len, const YeptrisParseOptions* opts,
-                                  int json_mode, YeptrisStatus* status);
+                                  int json_mode, int lazy, YeptrisStatus* status);
 
 YEPTRIS_API YeptrisDocument yeptris_parse(const char* buf, size_t len, YeptrisStatus* status) {
     return yeptris_parse_ex(buf, len, NULL, status);
@@ -47,6 +48,31 @@ yep_dom* yep_doc_dom(yeptris_document* doc) {
     if (doc == NULL) {
         return NULL;
     }
+    if (doc->dom == NULL && doc->lazy_tape != NULL && doc->lazy_kind == 1) {
+        /* #378: the YAML record tape — replay through the eager
+         * builders; the finish pool (whose spans the records carry)
+         * dies once the builders copied what must outlive it */
+        yep_ytape* t = (yep_ytape*)doc->lazy_tape;
+        yep_dom* dom = yep_dom_create(doc->sys);
+        if (dom != NULL) {
+            dom->resolver = (doc->schema == YEPTRIS_SCHEMA_11_COMPAT) ? yep_resolver_compat11()
+                                                                      : yep_resolver_core12();
+        }
+        if (dom != NULL && dom_from_ytape(dom, t) == 0) {
+            doc->dom = dom;
+            ytap_free(t);
+            yep_free(doc->sys, t);
+            doc->lazy_tape = NULL;
+            yep_pool_destroy((yep_pool*)doc->finish_pool);
+            doc->finish_pool = NULL;
+        } else {
+            yep_dom_destroy(dom);
+            dom = NULL; /* the tape stays: a later access retries, or
+                         * free drops it (replay fails only on
+                         * allocation/depth, as the builders do) */
+        }
+        return dom;
+    }
     if (doc->dom == NULL && doc->lazy_tape != NULL) {
         yeptris_json_tape* t = (yeptris_json_tape*)doc->lazy_tape;
         yep_dom* dom = yep_dom_create(doc->sys);
@@ -83,6 +109,21 @@ static int lazy_nums_settled(const yeptris_json_tape* t) {
         }
         uint32_t len = (uint32_t)((r >> 8) & 0x7FFFFFu);
         uint32_t off = (uint32_t)(r >> 32);
+        { /* the pure-integer fast path: all digits (optional '-'), no
+           * leading zero unless a lone digit — number_shape would
+           * consume the whole span and answer float-less; skip it */
+            const char* q = p + off;
+            size_t d = (*q == '-') ? 1 : 0;
+            size_t w = d;
+            for (; w < len; w++) {
+                if ((unsigned)(unsigned char)q[w] - '0' > 9u) {
+                    break;
+                }
+            }
+            if (w == len && len > d && (len - d == 1 || q[d] != '0')) {
+                continue;
+            }
+        }
         size_t adv = 0;
         int flt = 0;
         if (yep_json_number_shape(p + off, len, &adv, &flt) == 0 || adv != (size_t)len) {
@@ -107,6 +148,7 @@ static YeptrisDocument yep_json_doc_wrap(yep_dom* dom, const char* buf, size_t l
     doc->dom = dom;
     doc->sys = sys;
     doc->lazy_tape = NULL;
+    doc->lazy_kind = 0;
     doc->schema = YEPTRIS_SCHEMA_12_CORE; /* strict JSON is core by construction */
     doc->transcoded = NULL;
     doc->transcoded_len = 0;
@@ -161,6 +203,7 @@ YEPTRIS_API YeptrisDocument yeptris_parse_json(const char* buf, size_t len, Yept
                 YeptrisDocument h = yep_json_doc_wrap(NULL, buf, len, sys, status);
                 if (h != NULL) {
                     ((yeptris_document*)h)->lazy_tape = t;
+                    ((yeptris_document*)h)->lazy_kind = 0;
                     return h;
                 }
                 yeptris_tape_free(t); /* wrap failed (memory): below */
@@ -206,7 +249,7 @@ YEPTRIS_API YeptrisDocument yeptris_parse_json(const char* buf, size_t len, Yept
         }
         if (brc == -2) {
             yep_dom_destroy(dom);
-            return parse_impl(buf, len, NULL, 1, status);
+            return parse_impl(buf, len, NULL, 1, 0, status);
         }
         return yep_json_doc_wrap(dom, buf, len, sys, status);
     }
@@ -218,7 +261,7 @@ jfail:
 }
 
 static YeptrisDocument parse_impl(const char* buf, size_t len, const YeptrisParseOptions* opts,
-                                  int json_mode, YeptrisStatus* status) {
+                                  int json_mode, int lazy, YeptrisStatus* status) {
     YeptrisStatus st = YEPTRIS_OK;
     if (buf == NULL && len != 0) {
         st = YEPTRIS_ERROR_ARG;
@@ -303,24 +346,30 @@ engine_enter:
         st = YEPTRIS_ERROR_MEMORY;
         goto fail;
     }
-    yep_dom* dom = yep_dom_create(sys);
-    if (dom == NULL) {
-        yep_engine_destroy(eng);
-        yep_free(sys, transcoded);
-        st = YEPTRIS_ERROR_MEMORY;
-        goto fail;
+    /* the lazy route builds NO tree at parse: the dom (and its whole
+     * setup) exists only for the eager sink (the ASAN leak lesson —
+     * an unused dom leaked per lazy parse) */
+    yep_dom* dom = NULL;
+    if (!lazy) {
+        dom = yep_dom_create(sys);
+        if (dom == NULL) {
+            yep_engine_destroy(eng);
+            yep_free(sys, transcoded);
+            st = YEPTRIS_ERROR_MEMORY;
+            goto fail;
+        }
+        /* the direct builders resolve with the engine's schema (the typing
+         * SSOT — one resolver decision per scalar, whichever path builds) */
+        dom->resolver = (opts != NULL && opts->schema == YEPTRIS_SCHEMA_11_COMPAT)
+                            ? yep_resolver_compat11()
+                            : yep_resolver_core12();
+        /* BEFORE the run: input-slice strings borrow as views (zero copy)
+         * — the document keeps `transcoded` alive for exactly this. Set
+         * after, every scalar was arena-copied (found 2026-09-10). */
+        dom->input_len = data_len;
+        dom->input_base = transcoded ? (const char*)transcoded : buf;
+        yep_dom_prepare_len(dom, data_len);
     }
-    /* the direct builders resolve with the engine's schema (the typing
-     * SSOT — one resolver decision per scalar, whichever path builds) */
-    dom->resolver = (opts != NULL && opts->schema == YEPTRIS_SCHEMA_11_COMPAT)
-                        ? yep_resolver_compat11()
-                        : yep_resolver_core12();
-    /* BEFORE the run: input-slice strings borrow as views (zero copy)
-     * — the document keeps `transcoded` alive for exactly this. Set
-     * after, every scalar was arena-copied (found 2026-09-10). */
-    dom->input_len = data_len;
-    dom->input_base = transcoded ? (const char*)transcoded : buf;
-    yep_dom_prepare_len(dom, data_len);
     {
         /* the nametab reserve survives: a memchr chain for '&' is
          * far cheaper than the class sweep it replaced */
@@ -336,15 +385,31 @@ engine_enter:
         yep_engine_prepare(eng, &amp_only);
     }
 
-    yep_sink sink = {.on_event = yep_dom_on_event,
-                     .ctx = dom,
-                     .on_flow_build = dom_on_flow_build,
-                     .on_flow_commit = dom_on_flow_commit,
-                     .on_flow_rollback = dom_on_flow_rollback,
-                     .on_block_pair = dom_on_block_pair,
-                     .on_scalar = dom_on_scalar,
-                     .on_block_open = dom_on_block_open,
-                     .on_block_item = dom_on_block_item};
+    yep_ytape yt = {0}; /* zeroed: MSVC cannot couple have_tape to ytap_init's fill */
+    int have_tape = 0;
+    yep_sink sink;
+    if (lazy) {
+        /* #378: the recorder rides the SAME engine run — the fused
+         * arms stay hot, the builders' node lane defers to replay */
+        if (ytap_init(&yt, data, data_len, 0) != 0) {
+            yep_engine_destroy(eng);
+            yep_free(sys, transcoded);
+            st = YEPTRIS_ERROR_MEMORY;
+            goto fail;
+        }
+        ytap_sink(&yt, &sink);
+        have_tape = 1;
+    } else {
+        sink = (yep_sink){.on_event = yep_dom_on_event,
+                          .ctx = dom,
+                          .on_flow_build = dom_on_flow_build,
+                          .on_flow_commit = dom_on_flow_commit,
+                          .on_flow_rollback = dom_on_flow_rollback,
+                          .on_block_pair = dom_on_block_pair,
+                          .on_scalar = dom_on_scalar,
+                          .on_block_open = dom_on_block_open,
+                          .on_block_item = dom_on_block_item};
+    }
     int rc = yep_engine_run(eng, data, data_len, &sink);
     if (rc != 0) {
         const yep_error* ee = yep_engine_error(eng);
@@ -353,6 +418,9 @@ engine_enter:
             yep_error* tls = yep_error_tls();
             *tls = *ee;
         }
+        if (have_tape) {
+            ytap_free(&yt);
+        }
         yep_dom_destroy(dom);
         yep_engine_destroy(eng);
         yep_free(sys, transcoded);
@@ -366,12 +434,14 @@ engine_enter:
      * dangle at engine teardown (found by ASAN). */
     yep_pool* finish = yep_engine_detach_pool(eng);
     yep_engine_destroy(eng);
-    /* DOM strings are input-offsets or arena copies; nothing in the
-     * tree references the finish pool anymore — release it now */
 
     /* Empty stream (no documents): NULL document with YEPTRIS_OK. */
-    if (dom->dcount == 0) {
-        yep_dom_destroy(dom);
+    if (have_tape ? (yt.docs == 0) : (dom->dcount == 0)) {
+        if (have_tape) {
+            ytap_free(&yt);
+        } else {
+            yep_dom_destroy(dom);
+        }
         yep_pool_destroy(finish);
         yep_free(sys, transcoded);
         if (status != NULL) {
@@ -379,6 +449,13 @@ engine_enter:
         }
         return NULL;
     }
+    if (!have_tape) {
+        /* DOM strings are input-offsets or arena copies; nothing in the
+         * tree references the finish pool anymore — release it now */
+        yep_pool_destroy(finish);
+        finish = NULL;
+    } /* the lazy route keeps it: the records carry pool spans until
+         replay copies (doc.h's finish_pool contract) */
 
     yeptris_document* doc = yep_alloc(sys, sizeof(yeptris_document));
     if (doc == NULL) {
@@ -388,7 +465,6 @@ engine_enter:
         st = YEPTRIS_ERROR_MEMORY;
         goto fail;
     }
-    doc->dom = dom;
     doc->sys = sys;
     doc->schema = (opts != NULL && opts->schema == YEPTRIS_SCHEMA_11_COMPAT)
                       ? YEPTRIS_SCHEMA_11_COMPAT
@@ -396,12 +472,26 @@ engine_enter:
     doc->transcoded = transcoded;
     doc->transcoded_len = transcoded_len;
     doc->input = buf;
-    /* the finish pool dies here: every DOM string is an input offset
-     * or an arena copy (dom_ev_str) — nothing references it */
-    yep_pool_destroy(finish);
-    doc->finish_pool = NULL;
-    doc->lazy_tape = NULL; /* field-by-field ctor: leave no garbage
-                            * (document_free frees a non-NULL tape) */
+    doc->finish_pool = finish; /* NULL on the eager route */
+    doc->lazy_tape = NULL;
+    doc->lazy_kind = 0;
+    if (have_tape) {
+        yep_ytape* heap_tape = yep_alloc(sys, sizeof(*heap_tape));
+        if (heap_tape == NULL) {
+            ytap_free(&yt);
+            yep_pool_destroy(finish);
+            yep_free(sys, transcoded);
+            yep_free(sys, doc);
+            st = YEPTRIS_ERROR_MEMORY;
+            goto fail;
+        }
+        *heap_tape = yt;
+        doc->dom = NULL;
+        doc->lazy_tape = heap_tape;
+        doc->lazy_kind = 1;
+    } else {
+        doc->dom = dom;
+    }
     return (YeptrisDocument)doc;
 
 fail:
@@ -414,7 +504,15 @@ fail:
 YEPTRIS_API YeptrisDocument yeptris_parse_ex(const char* buf, size_t len,
                                              const YeptrisParseOptions* opts,
                                              YeptrisStatus* status) {
-    return parse_impl(buf, len, opts, 0, status);
+    return parse_impl(buf, len, opts, 0, 0, status);
+}
+
+/* #378 slice 1 (internal seam): the YAML parse rides the packed record
+ * tape — the tree materializes from the records on first access. Same
+ * grammar, same errors; parse-only workloads never build nodes. */
+YeptrisDocument yeptris_parse_ytape_ex(const char* buf, size_t len, const YeptrisParseOptions* opts,
+                                       YeptrisStatus* status) {
+    return parse_impl(buf, len, opts, 0, 1, status);
 }
 
 YEPTRIS_API void yeptris_document_free(YeptrisDocument handle) {
@@ -423,7 +521,11 @@ YEPTRIS_API void yeptris_document_free(YeptrisDocument handle) {
         return;
     }
     if (doc->lazy_tape != NULL) { /* never materialized: free stays free */
-        yeptris_tape_free((yeptris_json_tape*)doc->lazy_tape);
+        if (doc->lazy_kind == 1) {
+            ytap_free((yep_ytape*)doc->lazy_tape);
+        } else {
+            yeptris_tape_free((yeptris_json_tape*)doc->lazy_tape);
+        }
         yep_free(doc->sys, doc->lazy_tape);
     }
     yep_dom_destroy(doc->dom);

data/vendor/libyeptris/src/yeptris/scan/json.c

@@ -761,12 +761,13 @@ yep_jw_status yep_json_walk_next(yep_json_walk* w, yep_json_tok* t) {
  * parity error. The mask-level identity set (escape scanner via the
  * ODD_BITS borrow trick, prefix-xor string parity) is simdjson's. */
 
-YEPTRIS_API int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx) {
+YEPTRIS_API int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                                       unsigned* flags) {
     size_t n = 0;
-    uint64_t prev_in_string = 0, esc_carry = 0, follows_carry = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
     for (size_t off = 0; off < len; off += 64) {
         size_t cn = len - off < 64 ? len - off : 64;
-        uint64_t q = 0, bs = 0, op = 0, ws = 0;
+        uint64_t q = 0, bs = 0, op = 0, ws = 0, c0 = 0;
         for (size_t k = 0; k < cn; k++) {
             unsigned char c = (unsigned char)p[off + k];
             uint64_t bit = 1ull << k;
@@ -779,14 +780,51 @@ YEPTRIS_API int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx,
             } else if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
                 ws |= bit;
             }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
         }
         uint64_t valid = cn == 64 ? ~0ull : ((1ull << cn) - 1ull);
-        n = yep_json_stage1_resolve(q, bs, op, ws, valid, &prev_in_string, &esc_carry,
-                                    &follows_carry, off, idx, n);
+        n = yep_json_stage1_resolve(q, bs, op, ws, c0, valid, &prev_in_string, &esc_carry, off, idx,
+                                    n, flags);
     }
     *nidx = n;
     return prev_in_string ? 0 : 1;
 }
 
+YEPTRIS_API int yep_json_stage1_masks_scalar(const char* p, size_t len, struct yep_s1_block* blocks,
+                                             size_t* nblocks, unsigned* flags) {
+    size_t nb = 0;
+    uint64_t prev_in_string = 0, esc_carry = 0;
+    for (size_t off = 0; off < len; off += 64) {
+        size_t cn = len - off < 64 ? len - off : 64;
+        uint64_t q = 0, bs = 0, op = 0, c0 = 0;
+        for (size_t k = 0; k < cn; k++) {
+            unsigned char c = (unsigned char)p[off + k];
+            uint64_t bit = 1ull << k;
+            if (c == '"') {
+                q |= bit;
+            } else if (c == '\\') {
+                bs |= bit;
+            } else if (c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == ':') {
+                op |= bit;
+            }
+            if (c < 0x20) {
+                c0 |= bit;
+            }
+        }
+        uint64_t valid = cn == 64 ? ~0ull : ((1ull << cn) - 1ull);
+        blocks[nb].q = q;
+        blocks[nb].bs = bs;
+        blocks[nb].op = op;
+        blocks[nb].c0 = c0;
+        blocks[nb].tokens =
+            yep_json_stage1_tokens(q, bs, op, c0, valid, &prev_in_string, &esc_carry, flags);
+        nb++;
+    }
+    *nblocks = nb;
+    return prev_in_string ? 0 : 1;
+}
+
 /* The per-chunk classifier: the kernels table's json_chunk slot
  * (scalar reference in common, NEON/AVX2 TUs on their ISAs). */

data/vendor/libyeptris/src/yeptris/scan/json.h

@@ -93,8 +93,15 @@ YEPTRIS_API int yep_json_string(const char* p, size_t len, size_t* i, size_t* cl
  * (scalar reference; the kernels table's json_stage1 slot carries the
  * ISA twins). Writes the token positions stage 2 dispatches on —
  * operators, string OPEN quotes, scalar-run starts — and returns 0
- * only for an unterminated string. idx holds len+2 entries. */
-YEPTRIS_API int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx);
+ * only for an unterminated string. idx holds len+2 entries. flags
+ * (never NULL) accumulates the YEP_S1_* doc-level facts. */
+YEPTRIS_API int yep_json_stage1_scalar(const char* p, size_t len, uint32_t* idx, size_t* nidx,
+                                       unsigned* flags);
+/* The classification half (no emission) — the kernels table's
+ * json_stage1_masks slot carries the ISA twins. */
+struct yep_s1_block; /* defined in scan/json.c (the span blocks) */
+YEPTRIS_API int yep_json_stage1_masks_scalar(const char* p, size_t len, struct yep_s1_block* blocks,
+                                             size_t* nblocks, unsigned* flags);
 
 /* The per-chunk byte-class classifier behind the flow kernels: the
  * kernels table's json_chunk slot (common/simd_text.h) — yep_chunk_masks