yeptris 0.6.52.0-arm-linux-eabihf → 0.6.53.0-arm-linux-eabihf

data/vendor/libyeptris/src/yeptris/parse/engine.c

@@ -49,7 +49,7 @@ typedef enum {
 
 /* One open collection frame (the engine's indentation stack). */
 typedef struct {
-    uint16_t col;
+    uint32_t col;
     uint8_t kind;
     uint8_t inline_doc; /* opened on the --- line: deeper keys only */
 } yep_frame;
@@ -163,7 +163,7 @@ struct yep_engine {
     int last_root_flow;                                 /* a root-level flow node just completed */
     int tag_undef;                                      /* an unresolved !handle! was seen (QLJ7) */
     int doc_inline;                                     /* content node opened on the --- line */
-    uint16_t flow_floor; /* parent column of the current block-level flow */
+    uint32_t flow_floor; /* parent column of the current block-level flow */
     int flow_enforce;    /* flow continuation lines must out-indent it */
     int compat_grammar;  /* 11-COMPAT: libyaml/psych grammar parity —
                           * the flow indent floor (9C9N/VJP3) is a
@@ -176,8 +176,8 @@ struct yep_engine {
 
 /* ------------------------------------------------------------ forwards */
 
-static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col);
-static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col);
+static int e_node(yep_engine* e, yep_ctx ctx, uint32_t floor_col);
+static int e_parse_value(yep_engine* e, yep_ctx ctx, uint32_t floor_col);
 
 /* ---------------------------------------------------------------- util */
 
@@ -462,7 +462,7 @@ static uint32_t anchor_define(yep_engine* e, yep_view name) {
 /* Opens a collection frame unless the top frame already continues at
  * exactly this column (sibling entries reuse it). anchor/tag ride the
  * START event. Returns 0 on success. */
-static int e_open_seq(yep_engine* e, uint16_t col, uint32_t line, uint32_t coln, yep_view anchor,
+static int e_open_seq(yep_engine* e, uint32_t col, uint32_t line, uint32_t coln, yep_view anchor,
                       yep_view tag, uint32_t anchor_id) {
     if (e->depth > 0 && e->frames[e->depth - 1].kind == YEP_FRAME_SEQ &&
         e->frames[e->depth - 1].col == col) {
@@ -500,7 +500,7 @@ static int e_open_seq(yep_engine* e, uint16_t col, uint32_t line, uint32_t coln,
 
 /* silent: the sink's on_block_open built the node — push the frame
  * without emitting (TODO.restructure/57). */
-static int e_open_map(yep_engine* e, uint16_t col, uint32_t line, uint32_t coln, yep_view anchor,
+static int e_open_map(yep_engine* e, uint32_t col, uint32_t line, uint32_t coln, yep_view anchor,
                       yep_view tag, uint32_t anchor_id, int silent) {
     if (e->depth > 0 && e->frames[e->depth - 1].kind == YEP_FRAME_MAP &&
         e->frames[e->depth - 1].col == col) {
@@ -555,7 +555,7 @@ static int e_flush_q_value(yep_engine* e) {
  * value arrived inline) the innermost '?' pending resolves — without
  * this pop the stack keeps a stale entry and the outer pair's null
  * flushes inside the wrong map (the nightly's run 38387614146). */
-static int e_pair_value_done(yep_engine* e, yep_ctx ctx, uint16_t key_col) {
+static int e_pair_value_done(yep_engine* e, yep_ctx ctx, uint32_t key_col) {
     /* the pair's own level, BEFORE its value parse: the value may open
      * deeper frames ("? a:\n   ? b<EOF>") and a post-parse depth read
      * then matches the DEEPER pair's pending slot, popping the wrong
@@ -604,7 +604,7 @@ static int e_close_to(yep_engine* e, int to_depth) {
 /* ------------------------------------------------------------- scalars */
 
 /* Quoted scalar at the opening quote; spans lines when needed. */
-static int e_quoted_floor(yep_engine* e, yep_event* ev, uint16_t min_indent, int enforce) {
+static int e_quoted_floor(yep_engine* e, yep_event* ev, uint32_t min_indent, int enforce) {
     unsigned char q = (unsigned char)e->p[e->pos];
     size_t content_start = e->pos + 1;
     int has_esc = 0;
@@ -885,12 +885,12 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
                 if (content_indent <= parent_col) {
                     break; /* no block content */
                 }
-            } else if (li.indent <= parent_col) {
+            } else if ((int)li.indent <= parent_col) {
                 break; /* dedent ends the block */
-            } else if (li.indent < content_indent && li.indent <= parent_col) {
+            } else if ((int)li.indent < content_indent && (int)li.indent <= parent_col) {
                 break;
             }
-            if (li.indent < content_indent) {
+            if ((int)li.indent < content_indent) {
                 break;
             }
         }
@@ -902,10 +902,10 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
                     if (content_indent <= parent_col) {
                         break;
                     }
-                } else if (li.indent < content_indent) {
+                } else if ((int)li.indent < content_indent) {
                     break;
                 }
-                if (li.indent >= content_indent) {
+                if ((int)li.indent >= content_indent) {
                     uint32_t from2 = li.offset + (uint32_t)content_indent;
                     if (e->fold_n >= YEP_MAX_FOLD_LINES) {
                         return e_fail(e, YEP_ERR_MEMORY, e->pos);
@@ -945,7 +945,7 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
             }
             /* a blank line whose spaces extend past the content indent is
              * CONTENT (kept verbatim); otherwise it is a pure break */
-            if (content_indent >= 0 && li.indent > content_indent) {
+            if (content_indent >= 0 && (int)li.indent > content_indent) {
                 if (e->fold_n >= YEP_MAX_FOLD_LINES) {
                     return e_fail(e, YEP_ERR_MEMORY, e->pos);
                 }
@@ -996,7 +996,7 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
         /* leading blank lines belong to the block content */
         e->fold[e->fold_n].breaks_before = saw_content ? pending_breaks + 1 : pending_breaks;
         e->fold[e->fold_n].more_indented =
-            li.indent > content_indent ||
+            (int)li.indent > content_indent ||
             (from < li.end && (e->p[from] == ' ' || e->p[from] == '\t'));
         e->fold_n++;
         saw_content = 1;
@@ -1893,7 +1893,7 @@ static int e_flow_key_after(const char* p, size_t len, size_t close) {
  * rejects it, and the fast path must agree (the nightly's run
  * 38280372447: whole vs chunked split across the fast/slow paths and
  * produced two different verdicts for the same bytes). */
-static int e_flow_floor_ok(const char* p, size_t open, size_t close, uint16_t floor) {
+static int e_flow_floor_ok(const char* p, size_t open, size_t close, uint32_t floor) {
     size_t j = open;
     while (j < close) {
         /* the break rule is the scan's: \n, or a \r the next byte does
@@ -3036,7 +3036,7 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
 
 /* Plain (optionally anchored) value: emit, fold continuations, land at
  * the line end. root_ctx is 0 — every arm has pushed a frame. */
-static int e_shape_plain_value(yep_engine* e, const yep_line_shape* sh, uint16_t floor,
+static int e_shape_plain_value(yep_engine* e, const yep_line_shape* sh, uint32_t floor,
                                yep_view anchor, uint32_t anchor_id) {
     yep_event ev;
     e_event_init(&ev, YEP_EV_SCALAR);
@@ -3078,7 +3078,7 @@ static int e_shape_plain_value(yep_engine* e, const yep_line_shape* sh, uint16_t
 
 /* Flow value: the JSON fast path first; anything else restores the
  * cursor and runs the value chain (which re-derives the flow arms). */
-static int e_shape_flow_value(yep_engine* e, const yep_line_shape* sh, uint16_t floor,
+static int e_shape_flow_value(yep_engine* e, const yep_line_shape* sh, uint32_t floor,
                               yep_ctx next) {
     yep_view none = {NULL, 0};
     e->pos = sh->val_start;
@@ -3119,7 +3119,7 @@ static int e_shape_alias_value(yep_engine* e, const yep_line_shape* sh, int unde
         if (e_simple_key_ok(e, sh->val_start, e->pos) != 0) {
             return -1;
         }
-        uint16_t key_col = (uint16_t)e_col(e, sh->val_start);
+        uint32_t key_col = e_col(e, sh->val_start);
         int rc = e_open_map(e, key_col, e->line, key_col + 1, none, none, 0, 0);
         if (rc != 0) {
             return rc;
@@ -3141,7 +3141,7 @@ static int e_shape_alias_value(yep_engine* e, const yep_line_shape* sh, int unde
  * emit it: the pair path takes the whole line without events, and
  * building kv up front discarded ~48 stores per classified line
  * (TODO.restructure/63 phase 1). */
-static void e_key_event(const yep_engine* e, const yep_line_shape* sh, uint16_t key_col,
+static void e_key_event(const yep_engine* e, const yep_line_shape* sh, uint32_t key_col,
                         uint32_t line, yep_event* kv) {
     e_event_init(kv, YEP_EV_SCALAR);
     kv->style = YEP_STYLE_PLAIN;
@@ -3156,7 +3156,7 @@ static void e_key_event(const yep_engine* e, const yep_line_shape* sh, uint16_t
     e_pos_mark(e, sh->key_end, &kv->end_line, &kv->end_col);
 }
 
-static int e_classified(yep_engine* e, uint16_t floor_col) {
+static int e_classified(yep_engine* e, uint32_t floor_col) {
     yep_line_info li = e_line_info_here(e);
     if (e->pos != e->line_start + li.indent) {
         return 0; /* mid-line entry: the chain keeps its dispatch */
@@ -3177,7 +3177,7 @@ static int e_classified(yep_engine* e, uint16_t floor_col) {
         default:
             return 0;
         }
-        uint16_t col = (uint16_t)e_col(e, e->pos);
+        uint32_t col = e_col(e, e->pos);
         int rc = e_open_seq(e, col, e->line, col + 1, none, none, 0);
         if (rc != 0) {
             return rc;
@@ -3203,7 +3203,7 @@ static int e_classified(yep_engine* e, uint16_t floor_col) {
     if (e_simple_key_ok(e, sh->key_start, sh->key_end) != 0) {
         return -1;
     }
-    uint16_t key_col = (uint16_t)e_col(e, e->pos);
+    uint32_t key_col = e_col(e, e->pos);
     /* `key:` with the value on following lines, opening a FRESH map:
      * the sink may build map+key in one call (TODO.restructure/57) —
      * the engine then opens its frame silently. Sibling keys (the map
@@ -3241,7 +3241,7 @@ static int e_classified(yep_engine* e, uint16_t floor_col) {
     /* the pair facts are captured BEFORE the fold: e_plain_multiline
      * advances line/pos across continuations */
     uint32_t pair_line = e->line;
-    uint16_t pair_val_col = (uint16_t)e_col(e, sh->val_start);
+    uint32_t pair_val_col = e_col(e, sh->val_start);
     yep_block_value v;
     memset(&v, 0, sizeof(v));
     v.cls = (uint8_t)sh->val;
@@ -3375,7 +3375,7 @@ static int e_classified(yep_engine* e, uint16_t floor_col) {
  * pend props (parsed at the parent's value position on an earlier line)
  * belong to the VALUE node as a whole — a collection START event when
  * the value is a collection. */
-static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
+static int e_node(yep_engine* e, yep_ctx ctx, uint32_t floor_col) {
     size_t node_at = e->pos;
     yep_view pend_a = e->pend_anchor, pend_t = e->pend_tag;
     uint32_t pend_aid = e->pend_anchor_id;
@@ -3482,7 +3482,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
         if (!yep_view_is_empty(anchor) || !yep_view_is_empty(tag)) {
             return e_fail(e, YEP_ERR_UNEXPECTED, e->pos); /* "&a - x" */
         }
-        uint16_t col = (uint16_t)e_col(e, e->pos);
+        uint32_t col = e_col(e, e->pos);
         int rc = e_open_seq(e, col, e->line, col + 1, node_a, node_t, node_aid);
         if (rc != 0) {
             return rc;
@@ -3515,7 +3515,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
         if (e->pos > 0 && e->p[e->pos - 1] == '\t') {
             return e_fail(e, YEP_ERR_UNEXPECTED, e->pos); /* "?\t-" (Y79Y) */
         }
-        uint16_t col = (uint16_t)e_col(e, e->pos);
+        uint32_t col = e_col(e, e->pos);
         int rc = e_open_map(e, col, e->line, col + 1, pend_a, pend_t, pend_aid, 0);
         if (rc != 0) {
             return rc;
@@ -3639,7 +3639,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
             if (e_simple_key_ok(e, node_at, e->pos) != 0) {
                 return -1;
             }
-            uint16_t key_col = (uint16_t)e_col(e, node_at);
+            uint32_t key_col = e_col(e, node_at);
             int rc = e_open_map(e, key_col, e->line, key_col + 1, pend_a, pend_t, pend_aid, 0);
             if (rc != 0) {
                 return rc;
@@ -3683,7 +3683,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
             if (e_simple_key_ok(e, node_at, e->pos) != 0) {
                 return -1;
             }
-            uint16_t key_col = (uint16_t)e_col(e, node_at);
+            uint32_t key_col = e_col(e, node_at);
             int rc = e_open_map(e, key_col, e->line, key_col + 1, pend_a, pend_t, pend_aid, 0);
             if (rc != 0) {
                 return rc;
@@ -3722,7 +3722,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
             if (e_simple_key_ok(e, node_at, e->pos) != 0) {
                 return -1;
             }
-            uint16_t key_col = (uint16_t)e_col(e, node_at);
+            uint32_t key_col = e_col(e, node_at);
             int rc = e_open_map(e, key_col, e->line, key_col + 1, pend_a, pend_t, pend_aid, 0);
             if (rc != 0) {
                 return rc;
@@ -3777,7 +3777,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
         if (e->doc_inline && (!yep_view_is_empty(anchor) || !yep_view_is_empty(tag))) {
             return e_fail(e, YEP_ERR_UNEXPECTED, e->pos); /* "--- &a k: v" */
         }
-        uint16_t key_col = (uint16_t)e_col(e, node_at);
+        uint32_t key_col = e_col(e, node_at);
         int rc = e_open_map(e, key_col, e->line, key_col + 1, pend_a, pend_t, pend_aid, 0);
         if (rc != 0) {
             return rc;
@@ -3837,7 +3837,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
 }
 
 /* Value after "key:" / "-" / "?": on this line, or on following lines. */
-static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
+static int e_parse_value(yep_engine* e, yep_ctx ctx, uint32_t floor_col) {
     e_skip_inline_space(e);
     if (!e_at_eol(e)) {
         if (ctx == YEP_CTX_AFTER_COLON && e->p[e->pos] == '-' &&
@@ -3849,7 +3849,7 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
             (e->pos + 1 >= e->len || e->p[e->pos + 1] == ' ' || e->p[e->pos + 1] == '\t' ||
              e->p[e->pos + 1] == '\n' || e->p[e->pos + 1] == '\r')) {
             /* "- :" — a compact pair with an empty key */
-            uint16_t col = (uint16_t)e_col(e, e->pos);
+            uint32_t col = e_col(e, e->pos);
             int rc = e_open_map(e, col, e->line, col + 1, e->pend_anchor, e->pend_tag,
                                 e->pend_anchor_id, 0);
             if (rc != 0) {
@@ -4201,15 +4201,10 @@ yep_engine* yep_engine_create(const yep_allocator* sys) {
  * length is not strlen()'s: the INPUT may carry NUL bytes of its own,
  * and a strlen() of the copy truncates the run at the first one (the
  * nightly's run 38524928004: "\0\n---\r<U+2028>" whole-parsed as an
- * empty stream while the chunked cut skipped the copy). */
-static const char* e_normalize_breaks(yep_engine* e, const char* p, size_t len, size_t* out_len) {
-    char* out = yep_engine_normalize_breaks(e->sys, p, len, out_len, NULL);
-    if (out == NULL) {
-        return p;
-    }
-    e->norm_buf = out;
-    return out;
-}
+ * empty stream while the chunked cut skipped the copy). The context
+ * walk behind the copy lives below (issue #638); since #839 the
+ * chunked route threads it across --- cuts so every document
+ * normalizes to the same bytes the one-buffer walk produces. */
 
 /* The NEL/LS/PS -> '\n' copy (the one-shot path rides this BEFORE the
  * engine so the normalized bytes own the document's `transcoded` slot;
@@ -4258,6 +4253,13 @@ typedef struct yep_norm_ctx {
     int bs_content;
     uint32_t bs_indent; /* the header line's indent; (uint32_t)-1 = the
                          * doc-inline root rule (markers only) */
+    uint8_t started;    /* at least one byte consumed: the walk's i == 0
+                         * clauses are stream-start, not buffer-start —
+                         * the chunked route resumes mid-stream at a ---
+                         * cut, and the carried context must say so */
+    unsigned char prev; /* the last consumed byte (valid when started):
+                         * the '#' and line-start tests peek p[i - 1],
+                         * which at a resumed buffer is carried state */
 } yep_norm_ctx;
 
 typedef struct yep_norm_tok {
@@ -4415,8 +4417,12 @@ static yep_norm_tok norm_step(yep_norm_ctx* c, const char* p, size_t len, size_t
         norm_break_arms(c);
         return t;
     }
-    if (ch == '#' && (i == 0 || c->at_break || p[i - 1] == '\n' || p[i - 1] == '\r' ||
-                      p[i - 1] == ' ' || p[i - 1] == '\t')) {
+    /* the i == 0 clauses below are STREAM start: `started`/`prev` carry
+     * them across the chunked route's per-document resumes (issue #839)
+     * — a fresh context at a --- cut would otherwise read the resumed
+     * buffer's first byte as a line start the whole walk never saw */
+    if (ch == '#' && (!c->started || c->at_break || c->prev == '\n' || c->prev == '\r' ||
+                      c->prev == ' ' || c->prev == '\t')) {
         c->in_comment = 1;
         c->at_break = 0;
         return t;
@@ -4429,7 +4435,7 @@ static yep_norm_tok norm_step(yep_norm_ctx* c, const char* p, size_t len, size_t
         return t;
     }
 
-    int line_start = (i == 0 || c->at_break || p[i - 1] == '\n' || p[i - 1] == '\r');
+    int line_start = (!c->started || c->at_break || c->prev == '\n' || c->prev == '\r');
 
     /* block scalars: inside the fold no quote opens and every break
      * keeps its fold; the region ends at a col-0 marker, a %-line
@@ -4519,7 +4525,7 @@ static yep_norm_tok norm_step(yep_norm_ctx* c, const char* p, size_t len, size_t
     if (!c->bs_open && ch == '-' && i + 2 < len && p[i + 1] == '-' && p[i + 2] == '-' &&
         (i + 3 >= len || p[i + 3] == ' ' || p[i + 3] == '\t' || p[i + 3] == '\n' ||
          p[i + 3] == '\r') &&
-        (c->at_break || i == 0)) {
+        (c->at_break || !c->started)) {
         c->in_flow = 0; /* the marker bounds the document even when the
                          * walk still believes a flow is open (the same
                          * round-35 law as the in-quote rescue) */
@@ -4563,22 +4569,80 @@ static yep_norm_tok norm_step(yep_norm_ctx* c, const char* p, size_t len, size_t
         }
         c->tok_start = 0;
     }
-    if (ch == ',' || ch == ':' || ch == '[' || ch == '{' || ch == ']' || ch == '}') {
-        c->tok_start = 1; /* flow separators: the next byte may begin a
-                           * token with no space ("{\"a\":...") */
-    }
-    if (ch == '[' || ch == '{') {
+    if ((ch == '[' || ch == '{') && (c->tok_start || c->prev == ':')) {
+        /* a flow opener begins a token slot — at a token position, or
+         * directly after the value ':' ("k:{a: 1}" opens; the blank
+         * twin "k: {" rides tok_start already). Unconditioned, a
+         * plain's interior '[' ("ya200%[") opened a PHANTOM flow whose
+         * ',' armed the next token slot and a following "!<" opened a
+         * phantom verbatim — breaks inside verbatim ride unnormalized,
+         * the raw NELs defeated the line-facts kernel, and the plain
+         * fold rendered one fold-SPACE per break instead of the n-1
+         * newline tally (the r98 nightly — the #836 phantom's flow
+         * half; the comma half landed in #836 itself). */
+        c->tok_start = 1;
         c->in_flow++;
-    } else if (ch == ']' || ch == '}') {
+    } else if (ch == ',') {
+        /* flow separators glue the next token — THE COMMA IN FLOW
+         * ONLY. The unconditioned arm let a BLOCK plain's interior
+         * comma arm tok_start, a following "!<" opened a phantom
+         * verbatim tag that swallowed the rest of the input (breaks
+         * inside ride unnormalized), and the raw NEL/LS/PS bytes then
+         * defeated the line-facts kernel's \n/\r law — the plain
+         * fold's blank-line tally lost them (the r96 nightly,
+         * yeptris#836, "b,,,!<!<"+NEL^3). ':' stays unconditioned:
+         * the BLOCK value position rides it ("k: 'x'" opens the
+         * quote, the NormLsPs pins). */
         if (c->in_flow > 0) {
-            c->in_flow--;
+            c->tok_start = 1;
+        }
+    } else if (ch == ':') {
+        /* the separator law (#833's own predicate): a ':' arms the
+         * next token slot only before a blank/EOL/EOF — "k: 'x'"
+         * opens the value quote (the NormLsPs pins) while an
+         * interior "x:!<" of a block plain does NOT (the phantom
+         * verbatim again, the #836 colon-led twin) */
+        if (i + 1 >= len || p[i + 1] == ' ' || p[i + 1] == '\t' || p[i + 1] == '\n' ||
+            p[i + 1] == '\r') {
+            c->tok_start = 1;
         }
+    } else if (ch == ']' || ch == '}') {
+        c->tok_start = 1;
     }
     return t;
 }
 
+char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_t len,
+                                  size_t* out_len, int* kept_raw);
+
+static char* norm_breaks_run(const yep_allocator* sys, const char* p, size_t len, size_t* out_len,
+                             int* kept_raw, yep_norm_ctx* ctx);
+
 char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_t len,
                                   size_t* out_len, int* kept_raw) {
+    return norm_breaks_run(sys, p, len, out_len, kept_raw, NULL);
+}
+
+/* The context-threaded core (#839). ctx == NULL is the one-shot form:
+ * fresh state, and the !exotic early exit stands (no walk, no state).
+ * A non-NULL ctx rides in the step state: the chunked route parses
+ * whole documents, so the walk never sees a mid-construct cut — the
+ * divergence was the RESUME state. A --- cut lands exactly where the
+ * whole walk holds some carried state (an open quote it has not been
+ * rescued from, at_break, the raw prev byte); resuming fresh re-lexes
+ * the marker line differently ("--- 'tail": whole's rescue leaves
+ * tok_start 0 so the apostrophe is content, a fresh context's
+ * tok_start 1 opens a quote). Carrying, never resetting: doc N's walk
+ * sees the same bytes with the same state whole holds at the cut, so
+ * doc N + 1's start state equals whole's there — the induction makes
+ * the concatenated documents byte-identical to the one-buffer walk.
+ * The probe ALWAYS runs in this form (even without exotic leads): the
+ * carried state must advance over every byte, or the next resume
+ * starts stale. Both outcomes keep zero-copy semantics: a document
+ * whose exotic bytes all ride raw is used as-is, and those raw bytes
+ * are exactly what the one-buffer walk's copy carries for them. */
+static char* norm_breaks_run(const yep_allocator* sys, const char* p, size_t len, size_t* out_len,
+                             int* kept_raw, yep_norm_ctx* ctx) {
     if (kept_raw != NULL) {
         *kept_raw = 0;
     }
@@ -4595,15 +4659,21 @@ char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_
         exotic = 1; /* any NEL/LS/PS lead byte needs the context walk */
         break;
     }
-    if (!exotic) {
-        return NULL;
-    }
     yep_norm_ctx probe;
-    memset(&probe, 0, sizeof(probe));
-    probe.tok_start = 1;
+    if (ctx != NULL) {
+        probe = *ctx;
+    } else {
+        if (!exotic) {
+            return NULL;
+        }
+        memset(&probe, 0, sizeof(probe));
+        probe.tok_start = 1;
+    }
     int found = 0;
     for (size_t j = 0; j < len;) {
         yep_norm_tok t = norm_step(&probe, p, len, j);
+        probe.started = 1;
+        probe.prev = (unsigned char)p[j + t.n - 1];
         if (t.fold_off >= 0) {
             found = 1;
             break;
@@ -4611,22 +4681,34 @@ char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_
         j += t.n;
     }
     if (!found) {
-        if (kept_raw != NULL) {
-            *kept_raw = 1; /* every exotic byte rides raw */
+        if (ctx != NULL) {
+            *ctx = probe; /* the carried state advanced even with no fold */
+        } else {
+            if (kept_raw != NULL) {
+                *kept_raw = 1; /* every exotic byte rides raw */
+            }
+            return NULL; /* the exotic bytes all ride inside quoted regions:
+                          * the zero-copy path keeps them raw (the fix) */
         }
-        return NULL; /* the exotic bytes all ride inside quoted regions:
-                      * the zero-copy path keeps them raw (the fix) */
+        return NULL;
     }
     char* out = yep_alloc(sys, len + 1);
     if (out == NULL) {
         return NULL;
     }
     yep_norm_ctx cp;
-    memset(&cp, 0, sizeof(cp));
-    cp.tok_start = 1;
+    if (ctx != NULL) {
+        cp = *ctx; /* restart from the same carried state: the probe's
+                    * partial walk (it broke at the fold) is discarded */
+    } else {
+        memset(&cp, 0, sizeof(cp));
+        cp.tok_start = 1;
+    }
     size_t o = 0;
     for (size_t j = 0; j < len;) {
         yep_norm_tok t = norm_step(&cp, p, len, j);
+        cp.started = 1;
+        cp.prev = (unsigned char)p[j + t.n - 1];
         if (t.fold_off < 0) {
             if (kept_raw != NULL && *kept_raw == 0) {
                 for (size_t m = 0; m + 1 < t.n; m++) {
@@ -4652,6 +4734,19 @@ char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_
     if (out_len != NULL) {
         *out_len = o;
     }
+    if (ctx != NULL) {
+        *ctx = cp;
+    }
+    return out;
+}
+
+static const char* e_normalize_breaks(yep_engine* e, const char* p, size_t len, size_t* out_len,
+                                      yep_norm_ctx* ctx) {
+    char* out = norm_breaks_run(e->sys, p, len, out_len, NULL, ctx);
+    if (out == NULL) {
+        return p;
+    }
+    e->norm_buf = out;
     return out;
 }
 
@@ -4678,6 +4773,11 @@ typedef struct yep_stepstate {
     int at_break;       /* the previous position was a line break
                          * (multibyte breaks leave a non-break byte as
                          * p[i-1], so the '#' prev-byte test misses them) */
+    yep_norm_ctx nctx;  /* the break normalizer's carried context (#839):
+                         * each document's walk resumes with exactly the
+                         * state the whole-stream walk holds at the ---
+                         * cut, so per-document normalization is
+                         * byte-identical to the one-buffer walk */
 } yep_stepstate;
 
 static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep_sink* sink,
@@ -4834,6 +4934,19 @@ static size_t step_scan(yep_stepstate* st) {
                 st->tok_start = 1;
                 st->word_led = 0;
                 st->at_break = 1;
+            } else if (c == '\r' && (i + 1 >= st->len || (unsigned char)p[i + 1] != '\n')) {
+                /* a lone CR is a line break too (CRLF rides the \n arm
+                 * next iteration; the state is identical either way).
+                 * Missing this arm kept the scan inside a comment whose
+                 * line ended CR, the tab-led `{ {?` flow opener on the
+                 * next line went untracked, and the cut landed at the
+                 * following --- INSIDE the engine's open flow (the r97
+                 * nightly, yeptris#839: whole errored at the flow's ---
+                 * while chunked cut and errored at the cut edge) */
+                st->in_comment = 0;
+                st->tok_start = 1;
+                st->word_led = 0;
+                st->at_break = 1;
             } else if (c == 0xC2 && i + 1 < st->len && (unsigned char)p[i + 1] == 0x85) {
                 i++;
                 st->in_comment = 0;
@@ -5191,6 +5304,10 @@ int yep_engine_step(yep_engine* e, const char* chunk, size_t len, int final, con
     }
     yep_stepstate* st = e->step;
     st->tok_start = 1; /* the buffer starts at a line start */
+    if (st->nctx.started == 0) {
+        st->nctx.tok_start = 1; /* the stream's first walk: the same
+                                 * fresh state the one-shot starts */
+    }
     if (len > 0) {
         if (st->len + len > st->cap) {
             size_t cap = st->cap ? st->cap : 4096;
@@ -5256,7 +5373,8 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
      * it, and the next transcode replaces it. */
     e->parsing_final = emit_end;
     size_t norm_len = len;
-    e->p = e_normalize_breaks(e, buf, len, &norm_len);
+    yep_norm_ctx* nctx = e->step != NULL ? &((yep_stepstate*)e->step)->nctx : NULL;
+    e->p = e_normalize_breaks(e, buf, len, &norm_len, nctx);
     e->len = norm_len;
     e->pos = 0;
     e->line = 1 + e->line_base;
@@ -5314,7 +5432,7 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
                 e_fail(e, YEP_ERR_TAB_IN_INDENT, e->pos + li.indent);
                 goto fail;
             }
-            li.indent = (uint16_t)(t - li.offset);
+            li.indent = (uint32_t)(t - li.offset);
         }
         if (li.flags & YEP_LF_DIRECTIVE) {
             if (doc_open) {
@@ -5634,7 +5752,7 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
          * dash-blank test is computed once per line (TODO.restructure/75
          * — it was spelled out twice below); the top frame loads once
          * per iteration. */
-        uint16_t c = li.indent;
+        uint32_t c = li.indent;
         const int dash_blank =
             li.first == '-' &&
             (li.offset + li.indent + 1 >= e->len || e->p[li.offset + li.indent + 1] == ' ' ||

data/vendor/libyeptris/src/yeptris/parse/events.h

@@ -90,13 +90,13 @@ typedef struct yep_sink {
      * placed, 0 = not handled (the engine emits the two events
      * exactly as before), <0 = abort. */
     int (*on_block_pair)(void* ctx, const yep_view* key, const struct yep_block_value* v,
-                         uint32_t line, uint16_t key_col, uint16_t val_col);
+                         uint32_t line, uint32_t key_col, uint32_t val_col);
     /* Optional block-open fast path (TODO.restructure/57): a `key:`
      * line whose value lives on FOLLOWING lines, when the engine is
      * about to open a FRESH mapping. 1 = the sink built the map node
      * (placed, stack pushed) and the key node — the engine opens its
      * frame silently and skips both events; 0 = events as before. */
-    int (*on_block_open)(void* ctx, const yep_view* key, uint32_t line, uint16_t key_col);
+    int (*on_block_open)(void* ctx, const yep_view* key, uint32_t line, uint32_t key_col);
     /* Optional dash-item fast path (TODO.restructure/78): a classified
      * `- value` plain item after the engine opened (or continued) the
      * block sequence. 1 = the sink built and placed the item node,

data/vendor/libyeptris/src/yeptris/resolve/compat11.c

@@ -248,7 +248,8 @@ static yep_tag_id compat11(void* ctx, const char* p, uint32_t n) {
     if (n - i > 2 && p[i] == '0' && (p[i + 1] == 'b' || p[i + 1] == 'B')) {
         return is_digits_us(p + i + 2, n - i - 2) ? 1 : 0;
     }
-    if (n - i > 2 && p[i] == '0' && (p[i + 1] == 'x' || p[i + 1] == 'X')) {
+    if (n - i > 2 && p[i] == '0' &&
+        p[i + 1] == 'x') { /* lowercase only — psych keeps "0X1A" a String */
         uint32_t k = i + 2;
         if (p[k] == '_') {
             k++;

data/vendor/libyeptris/src/yeptris/resolve/core12.c

@@ -78,7 +78,9 @@ static yep_tag_id core12(void* ctx, const char* p, uint32_t n) {
             }
         }
     }
-    if (n - i > 2 && p[i] == '0' && (p[i + 1] == 'x' || p[i + 1] == 'X')) {
+    if (n - i > 2 && p[i] == '0' &&
+        p[i + 1] == 'x') { /* lowercase only: the core schema's int regexp; "0X1A" rides a string
+                              (psych agrees) */
         for (uint32_t k = i + 2; k < n; k++) {
             char c = p[k];
             if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) {

data/vendor/libyeptris/src/yeptris/scan/json.c

@@ -250,6 +250,9 @@ int yep_json_number_scan(const char* p, size_t len, size_t* i, int* is_float, in
                          double* dv) {
     size_t k = *i;
     int neg = 0;
+    if (k >= len) {
+        return 0; /* the caller's index is at the end: p[len] is OOB */
+    }
     if (p[k] == '-') {
         neg = 1;
         k++;
@@ -438,6 +441,9 @@ int yep_json_number(const char* p, size_t len, size_t* i) {
  * the full scan (the 2M tape-diff oracle pins the equivalence). */
 int yep_json_number_shape(const char* p, size_t len, size_t* i, int* is_float) {
     size_t k = *i;
+    if (k >= len) {
+        return 0; /* the caller's index is at the end: p[len] is OOB */
+    }
     if (p[k] == '-') {
         k++;
     }