yeptris 0.6.34.2 → 0.6.35.1

data/vendor/libyeptris/src/yeptris/parse/engine.c

@@ -130,11 +130,18 @@ struct yep_engine {
         yep_view handle, prefix;
     } tagmap[8];
     int tagmap_n;
-    int saw_yaml;     /* %YAML seen for the pending document */
-    size_t doc_begin; /* #179: first directive line of the pending doc
-                       * (SIZE_MAX = none) — libyaml's DOCUMENT_START
-                       * start_mark includes directives */
-    void* step;       /* yep_stepstate: resumable stepping (07) */
+    int saw_yaml;            /* %YAML seen for the pending document */
+    size_t doc_begin;        /* #179: first directive line of the pending doc
+                              * (SIZE_MAX = none) — libyaml's DOCUMENT_START
+                              * start_mark includes directives. An offset into
+                              * THIS run's buffer. */
+    uint32_t doc_begin_line; /* the same directive line as a stream-
+                              * absolute LINE: offsets die at a step cut
+                              * (the tail is memmoved), but a directive
+                              * line split from its --- must still mark
+                              * the DOCUMENT_START (#645's neighbor:
+                              * whole 85:1 vs chunked 86:1) */
+    void* step;              /* yep_stepstate: resumable stepping (07) */
 
     /* Flow single-pair deferral: a sequence entry's events are buffered
      * until we know whether ':' follows ("[a: b]" needs MAP_START before
@@ -1224,8 +1231,12 @@ static yep_view e_resolve_tag(yep_engine* e, yep_view tag) {
     return e_tag_uri_decode(e, e_resolve_tag_raw(e, tag));
 }
 
-/* Parses &anchor / !tag runs at the cursor into *anchor / *tag. */
+/* Parses &anchor / !tag runs at the cursor into *anchor / *tag. A node
+ * carries at most one of each (c-ns-properties): a second property of
+ * the same kind is the suite's 4JVG error. Keeping the last one let a
+ * mutated anchor run read differently from libyaml (catch 156). */
 static void e_props(yep_engine* e, yep_view* anchor, yep_view* tag) {
+    int have_anchor = 0, have_tag = 0;
     for (;;) {
         e_skip_inline_space(e);
         if (e->pos >= e->len) {
@@ -1233,6 +1244,11 @@ static void e_props(yep_engine* e, yep_view* anchor, yep_view* tag) {
         }
         unsigned char c = (unsigned char)e->p[e->pos];
         if (c == '&') {
+            if (have_anchor) {
+                e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
+                return;
+            }
+            have_anchor = 1;
             size_t start = e->pos + 1;
             e->pos = yep_scan_prop_end(e->p, e->len, start);
             anchor->p = e->p + start;
@@ -1257,6 +1273,11 @@ static void e_props(yep_engine* e, yep_view* anchor, yep_view* tag) {
                 return;
             }
         } else if (c == '!') {
+            if (have_tag) {
+                e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
+                return;
+            }
+            have_tag = 1;
             size_t start = e->pos;
             e->pos++;
             if (e->pos < e->len && e->p[e->pos] == '<') {
@@ -1478,12 +1499,18 @@ static int e_skip_flow(yep_engine* e) {
         }
         if (c == '[') {
             if (n >= e->max_depth) {
-                return e_fail(e, YEP_ERR_DEPTH, e->pos);
+                /* NOT-A-KEY deferral (the unterminated-quote law
+                 * above): the pre-scan fires before the flow's open
+                 * events, while the chunked feed had already streamed
+                 * them, so the whole stream lost MAP_START/SEQ_START
+                 * (the nightly's run 39502647277). The real parse owns
+                 * the depth error after the events. */
+                return 0;
             }
             stk[n++] = 0;
         } else if (c == '{') {
             if (n >= e->max_depth) {
-                return e_fail(e, YEP_ERR_DEPTH, e->pos);
+                return 0; /* the same deferral */
             }
             stk[n++] = 1;
         } else if (c == ']') {
@@ -2098,6 +2125,8 @@ static void e_flow_frame_init(void* stv, int idx, int kind, int pending_key) {
         uint8_t just_closed;
         uint8_t pair_open;
         uint8_t pair_value;
+        uint8_t pair_q; /* the pair opened on an explicit '?': its
+                         * JSON-style ':' separator is still ahead */
         uint8_t pair_wrap;
         uint8_t q_coll;   /* seq: an explicit '?' key collection awaits completion */
         uint8_t key_coll; /* map: the nested collection opened in key position */
@@ -2113,6 +2142,7 @@ static void e_flow_frame_init(void* stv, int idx, int kind, int pending_key) {
     st[idx].key_coll = 0;
     st[idx].pair_open = 0;
     st[idx].pair_value = 0;
+    st[idx].pair_q = 0;
     st[idx].pair_wrap = 0;
     st[idx].q_coll = 0;
     st[idx].buf_from = -1;
@@ -2128,6 +2158,8 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
         uint8_t just_closed; /* a nested collection closed here (key candidate) */
         uint8_t pair_open;   /* seq: a buffered single-pair awaits its MAP_END */
         uint8_t pair_value;  /* seq: the open pair already has its value */
+        uint8_t pair_q;      /* seq: the pair opened on an explicit '?':
+                              * its JSON-style ':' separator is still ahead */
         uint8_t pair_wrap;   /* map frame pushed as a seq single-pair */
         uint8_t q_coll;      /* seq: an explicit '?' key collection awaits completion */
         uint8_t key_coll;    /* map: a nested collection opened in key position */
@@ -2191,6 +2223,31 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
                 return e_fail(e, YEP_ERR_BAD_INDENT, e->pos + fl.indent);
             }
         }
+        if (st[n - 1].pair_open && st[n - 1].pair_q && e->pos < e->len && e->p[e->pos] == ':') {
+            /* "#671: the explicit key's entry separator may sit flush
+             * against the value ("[? &a :b]") -- the JSON-style colon
+             * the map frame's pending-key law consumes; an IMPLICIT
+             * key's separator is already spent here, so its value's
+             * leading ':' stays content ("[s: :v]") */
+            size_t n2 = e->pos + 1;
+            if (n2 >= e->len ||
+                !(e->p[n2] == ' ' || e->p[n2] == '\t' || e->p[n2] == '\n' || e->p[n2] == '\r' ||
+                  yep_ct_is((unsigned char)e->p[n2], YEP_CT_FLOW_IND))) {
+                st[n - 1].pair_q = 0;
+                e->pos++;
+                continue;
+            }
+        }
+        if (st[n - 1].kind == 1 && st[n - 1].pending_key == 2 && e->pos < e->len &&
+            e->p[e->pos] == ':') {
+            /* an explicit '?'-key entry's separator may follow on a
+             * LATER line (s-separate spans breaks in flow: "{?{a}\n:
+             * b}" — the nightly's catch 161); the post-node colon law
+             * never sees a colon at the node start */
+            st[n - 1].pending_key = 1;
+            e->pos++;
+            continue;
+        }
         if (e->pos < e->len && e->p[e->pos] != ',' && e->p[e->pos] != ']' && e->p[e->pos] != '}' &&
             !(e->p[e->pos] == ':' &&
               ((st[n - 1].kind == 1 && st[n - 1].pending_key == 1) || st[n - 1].just_closed)) &&
@@ -2389,6 +2446,7 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
                             return -2;
                         }
                         st[n - 1].pair_open = 1;
+                        st[n - 1].pair_q = (uint8_t)q_key;
                         continue;
                     } else {
                         yep_event mk;
@@ -2420,6 +2478,7 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
                         return -2;
                     }
                     st[n - 1].pair_open = 1;
+                    st[n - 1].pair_q = (uint8_t)q_key;
                     if (e->ev_scopes == 0 && e->ev_buf_n > 0) {
                         int brc = e_buf_send(e, 0);
                         if (brc != 0) {
@@ -2500,6 +2559,7 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
                 }
                 st[n - 1].pair_open = 0;
                 st[n - 1].pair_value = 0;
+                st[n - 1].pair_q = 0;
             }
             if (st[n - 1].buf_from >= 0) {
                 st[n - 1].buf_from = -1;
@@ -2739,6 +2799,12 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
                     }
                 }
                 st[n - 1].pair_open = 1;
+                /* the wrapped key's own open marked pair_value=1 ("a
+                 * VALUE collection completes the pending pair"); as a
+                 * KEY it completes nothing — the pair awaits a fresh
+                 * value, else the boundary flush skips the null ("[[a]:
+                 * ]" lost it, the nightly's round 9) */
+                st[n - 1].pair_value = 0;
                 continue; /* value next */
             }
             if (st[n - 1].kind == 0 && n >= 1) {
@@ -3710,8 +3776,9 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
          * may align with the '?' column. Flow nodes and property runs
          * continue the value even at the parent column ("k:\n!!seq\n[a]"). */
         if (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first == '-' &&
-            li.offset + 1 < li.end && e->p[li.offset + 1] != ' ' && e->p[li.offset + 1] != '\t' &&
-            e->p[li.offset + 1] != '\n' && e->p[li.offset + 1] != '\r') {
+            li.offset + li.indent + 1 < li.end && e->p[li.offset + li.indent + 1] != ' ' &&
+            e->p[li.offset + li.indent + 1] != '\t' && e->p[li.offset + li.indent + 1] != '\n' &&
+            e->p[li.offset + li.indent + 1] != '\r') {
             /* "-:" and friends: a dash with a non-space follower is a
              * PLAIN key ("-" => null), never the pending key's content
              * ("?\n-:" lost the null pair, the nightly's run
@@ -3756,9 +3823,33 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
                 goto empty_value;
             }
         }
+        /* A flow collection at the parent column is normally the pending
+         * value ("k:\n[a]"), but a flow followed by a same-line ':' is a
+         * KEY of the parent map: the pending value is empty and the main
+         * loop must own the line (crash-c0169e98: "a:\n{----}: 1" made
+         * the flow a's value and then errored on its key colon). Probe
+         * with the same pre-scan the block flow-key branch uses. */
+        int flow_value = 0;
+        if ((li.first == '[' || li.first == '{') && li.indent == floor_col) {
+            flow_value = 1;
+            size_t fsave = e->pos;
+            uint32_t fline = e->line;
+            size_t fls = e->line_start;
+            e->pos = li.offset + li.indent;
+            if (e_skip_flow(e) == 0) {
+                e_skip_inline_space(e);
+                if (e_colon_at(e, e->pos) && e->line == fline) {
+                    flow_value = 0;
+                }
+            } else {
+                yep_error_clear(&e->err); /* e_node re-raises the real error */
+            }
+            e->pos = fsave;
+            e->line = fline;
+            e->line_start = fls;
+        }
         if (li.indent > floor_col || e->depth == 0 ||
-            (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first != ':') ||
-            ((li.first == '[' || li.first == '{') && li.indent == floor_col) ||
+            (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first != ':') || flow_value ||
             /* libyaml never validates a block-scalar HEADER's column —
              * only the content must out-indent the parent (e_block_scalar
              * enforces that), so "|" / ">" at the key's own column is the
@@ -3941,48 +4032,366 @@ const yep_stopset yep_norm_break_leads = {
             0x02, 0x00}},
 };
 
+/* The context walk behind the break normalizer (issue #638): the fold
+ * is quote-aware — NEL (C2 85) folds EVERYWHERE (libyaml's READ_LINE
+ * folds it inside quoted scalars too), LS/PS (E2 80 A8/A9) fold
+ * everywhere EXCEPT inside single/double-quoted regions, where libyaml
+ * keeps them raw as content (the JSON suite's y_string_u+2028_line_sep
+ * / y_string_u+2029_par_sep demand it). The arms mirror step_scan's
+ * context tracking minus the cut/hold machinery — complete input, no
+ * truncation points — with two engine-faithful corrections: the value
+ * position survives spaces (`k: 'x'` opens the quote; a block plain
+ * continuation `a 'b'` does not), and the block-scalar floor is the
+ * header LINE's indent (step_scan's walk-back only serves col-0
+ * headers). Verified against libyaml by the NormLsPs pins and the
+ * fuzz-diff LS/PS comparator. */
+static int step_bs_header(const char* p, size_t len, size_t i);
+
+typedef struct yep_norm_ctx {
+    int quote; /* 0 none, else the opening quote char */
+    int in_comment;
+    int in_verbatim;
+    int in_flow;
+    int tok_start;
+    int word_led;
+    int at_break;
+    int bs_open;
+    int bs_content;
+    uint32_t bs_indent; /* the header line's indent; (uint32_t)-1 = the
+                         * doc-inline root rule (markers only) */
+} yep_norm_ctx;
+
+typedef struct yep_norm_tok {
+    size_t n;     /* bytes consumed */
+    int fold_off; /* -1 verbatim, else the offset inside the token
+                   * whose bytes become one '\n' */
+    int fold_len;
+} yep_norm_tok;
+
+static void norm_break_arms(yep_norm_ctx* c) {
+    c->tok_start = 1;
+    c->word_led = 0;
+    c->at_break = 1;
+}
+
+static yep_norm_tok norm_step(yep_norm_ctx* c, const char* p, size_t len, size_t i) {
+    yep_norm_tok t = {1, -1, 0};
+    unsigned char ch = (unsigned char)p[i];
+    int nel = (ch == 0xC2 && i + 1 < len && (unsigned char)p[i + 1] == 0x85);
+    int lps = (ch == 0xE2 && i + 2 < len && (unsigned char)p[i + 1] == 0x80 &&
+               ((unsigned char)p[i + 2] & 0xFE) == 0xA8);
+
+    if (c->in_comment) {
+        /* the folds end a comment like any line break (step_scan's
+         * law: the post-break '#' must read as a comment) */
+        if (ch == '\n') {
+            c->in_comment = 0;
+            norm_break_arms(c);
+        } else if (ch == '\r' && (i + 1 >= len || p[i + 1] != '\n')) {
+            c->in_comment = 0;
+            norm_break_arms(c);
+        } else if (nel) {
+            t.n = 2;
+            t.fold_off = 0;
+            t.fold_len = 2;
+            c->in_comment = 0;
+            norm_break_arms(c);
+        } else if (lps) {
+            t.n = 3;
+            t.fold_off = 0;
+            t.fold_len = 3;
+            c->in_comment = 0;
+            norm_break_arms(c);
+        }
+        return t;
+    }
+    if (c->quote != 0) {
+        /* NEL folds inside quotes too; LS/PS stay raw content */
+        if (nel) {
+            t.n = 2;
+            t.fold_off = 0;
+            t.fold_len = 2;
+            return t;
+        }
+        if (lps) {
+            t.n = 3;
+            return t;
+        }
+        if (c->quote == '\'') {
+            if (ch == '\'') {
+                if (i + 1 < len && p[i + 1] == '\'') {
+                    t.n = 2; /* '' escaped quote */
+                    return t;
+                }
+                c->quote = 0;
+            }
+            return t;
+        }
+        if (ch == '\\') {
+            /* the escape consumes the next byte — but a multibyte
+             * break under the escape still folds (an escaped break is
+             * a fold continuation, and the blind pass folded it) */
+            if (i + 1 >= len) {
+                return t;
+            }
+            unsigned char nx = (unsigned char)p[i + 1];
+            if (nx == 0xC2 && i + 2 < len && (unsigned char)p[i + 2] == 0x85) {
+                t.n = 3;
+                t.fold_off = 1;
+                t.fold_len = 2;
+                return t;
+            }
+            if (nx == 0xE2 && i + 3 < len && (unsigned char)p[i + 2] == 0x80 &&
+                ((unsigned char)p[i + 3] & 0xFE) == 0xA8) {
+                t.n = 4;
+                t.fold_off = 1;
+                t.fold_len = 3;
+                return t;
+            }
+            t.n = 2;
+            return t;
+        }
+        if (ch == '"') {
+            c->quote = 0;
+        }
+        return t;
+    }
+    if (c->in_verbatim) {
+        if (ch == '>') {
+            c->in_verbatim = 0;
+        }
+        return t;
+    }
+    if (ch == '\n' || (ch == '\r' && (i + 1 >= len || p[i + 1] != '\n'))) {
+        norm_break_arms(c);
+        return t;
+    }
+    if (nel) {
+        t.n = 2;
+        t.fold_off = 0;
+        t.fold_len = 2;
+        norm_break_arms(c);
+        return t;
+    }
+    if (lps) {
+        t.n = 3;
+        t.fold_off = 0;
+        t.fold_len = 3;
+        norm_break_arms(c);
+        return t;
+    }
+    if (ch == '#' && (i == 0 || c->at_break || p[i - 1] == '\n' || p[i - 1] == '\r' ||
+                      p[i - 1] == ' ' || p[i - 1] == '\t')) {
+        c->in_comment = 1;
+        c->at_break = 0;
+        return t;
+    }
+    if (ch == ' ' || ch == '\t') {
+        /* a value position survives spaces in block context ("k: 'x'"
+         * opens; the plain continuation "a 'b'" does not — its word
+         * did not begin with an indicator) */
+        c->tok_start = (c->in_flow > 0) ? 1 : (c->word_led || c->tok_start);
+        return t;
+    }
+
+    int line_start = (i == 0 || c->at_break || p[i - 1] == '\n' || p[i - 1] == '\r');
+
+    /* block scalars: inside the fold no quote opens and every break
+     * keeps its fold; the region ends at a col-0 marker, a %-line
+     * after content, or a dedent to the header line's indent */
+    if (line_start && c->in_flow == 0 && c->bs_open) {
+        size_t k = i;
+        uint32_t ind = 0;
+        while (k < len && p[k] == ' ') {
+            ind++;
+            k++;
+        }
+        if (k >= len) {
+            t.n = len - i;
+            c->at_break = 0;
+            return t; /* a final line without its break */
+        }
+        if (p[k] == '\n' || p[k] == '\r' || p[k] == '#') {
+            size_t e = k;
+            while (e < len && p[e] != '\n') {
+                e++;
+            }
+            t.n = (e < len) ? (e - i + 1) : (len - i);
+            c->at_break = (e < len);
+            return t; /* blank or comment-content line */
+        }
+        int marker = (ind == 0 && k + 2 < len &&
+                      ((p[k] == '-' && p[k + 1] == '-' && p[k + 2] == '-') ||
+                       (p[k] == '.' && p[k + 1] == '.' && p[k + 2] == '.')) &&
+                      (k + 3 >= len || p[k + 3] == ' ' || p[k + 3] == '\t' || p[k + 3] == '\n' ||
+                       p[k + 3] == '\r'));
+        if (marker || (p[k] == '%' && ind == 0 && c->bs_content) ||
+            (!marker && c->bs_indent != (uint32_t)-1 && ind <= (uint32_t)c->bs_indent)) {
+            c->bs_open = 0; /* the fold ends: this line processes normally */
+        } else {
+            size_t e = k;
+            while (e < len && p[e] != '\n') {
+                e++;
+            }
+            if (p[k] != '#' && p[k] != '\n' && p[k] != '\r') {
+                c->bs_content = 1;
+            }
+            t.n = (e < len) ? (e - i + 1) : (len - i);
+            c->at_break = (e < len);
+            return t;
+        }
+    }
+
+    /* the header: a '>' / '|' at a value position with a valid
+     * follower opens the fold (the engine's own floor is the parent's
+     * column, which the walk cannot know — the header LINE's indent
+     * bounds it from above and the dedent law agrees wherever the
+     * parse succeeds) */
+    if (line_start && c->in_flow == 0 && !c->bs_open && c->tok_start && (ch == '>' || ch == '|') &&
+        (i + 1 >= len || p[i + 1] == ' ' || p[i + 1] == '\t' || p[i + 1] == '\n' ||
+         p[i + 1] == '\r' || (p[i + 1] >= '0' && p[i + 1] <= '9') || p[i + 1] == '+' ||
+         p[i + 1] == '-')) {
+        int hdr = step_bs_header(p, len, i);
+        if (hdr > 0) {
+            size_t ls = i;
+            while (ls > 0 && p[ls - 1] != '\n' && p[ls - 1] != '\r') {
+                ls--;
+            }
+            uint32_t ind = 0;
+            size_t k = ls;
+            while (k < len && p[k] == ' ') {
+                ind++;
+                k++;
+            }
+            c->bs_open = 1;
+            c->bs_indent = ind;
+            c->bs_content = 0;
+            size_t e = i;
+            while (e < len && p[e] != '\n') {
+                e++;
+            }
+            t.n = (e < len) ? (e - i + 1) : (len - i);
+            c->at_break = (e < len);
+            return t;
+        }
+        /* not a header ("| : [" is a mapping key): fall through */
+    }
+    /* the doc-inline twin ("--- |"): the node hangs off the document,
+     * so only a later marker ends its fold (col-0 lines are content) */
+    if (line_start && c->in_flow == 0 && !c->bs_open && ch == '-' && i + 2 < len &&
+        p[i + 1] == '-' && p[i + 2] == '-' &&
+        (i + 3 >= len || p[i + 3] == ' ' || p[i + 3] == '\t' || p[i + 3] == '\n' ||
+         p[i + 3] == '\r')) {
+        size_t k = i + 3;
+        while (k < len && (p[k] == ' ' || p[k] == '\t')) {
+            k++;
+        }
+        if (k < len && (p[k] == '>' || p[k] == '|') &&
+            (k + 1 >= len || p[k + 1] == ' ' || p[k + 1] == '\t' || p[k + 1] == '\n' ||
+             p[k + 1] == '\r' || (p[k + 1] >= '0' && p[k + 1] <= '9') || p[k + 1] == '+' ||
+             p[k + 1] == '-') &&
+            step_bs_header(p, len, k) > 0) {
+            c->bs_open = 1;
+            c->bs_indent = (uint32_t)-1;
+            c->bs_content = 0;
+            size_t e = i;
+            while (e < len && p[e] != '\n') {
+                e++;
+            }
+            t.n = (e < len) ? (e - i + 1) : (len - i);
+            c->at_break = (e < len);
+            return t;
+        }
+        t.n = 3; /* the marker rides; the line's tail processes normally */
+        return t;
+    }
+
+    if (c->tok_start) {
+        c->at_break = 0; /* any content byte ends the line-start state */
+        if (ch == '!' && i + 1 < len && p[i + 1] == '<') {
+            c->in_verbatim = 1;
+            t.n = 2;
+            return t;
+        }
+        c->word_led = (ch == '&' || ch == '*' || ch == '!' || ch == '-' || ch == '?' || ch == ':' ||
+                       ch == ',' || ch == '[' || ch == '{' || ch == ']' || ch == '}');
+        if (ch == '\'' || ch == '"') {
+            c->quote = ch; /* a quote at a value position IS a quoted token */
+            c->tok_start = 0;
+            return t;
+        }
+        c->tok_start = 0;
+    }
+    if (ch == ',' || ch == ':' || ch == '[' || ch == '{' || ch == ']' || ch == '}') {
+        c->tok_start = 1; /* flow separators: the next byte may begin a
+                           * token with no space ("{\"a\":...") */
+    }
+    if (ch == '[' || ch == '{') {
+        c->in_flow++;
+    } else if (ch == ']' || ch == '}') {
+        if (c->in_flow > 0) {
+            c->in_flow--;
+        }
+    }
+    return t;
+}
+
 char* yep_engine_normalize_breaks(const yep_allocator* sys, const char* p, size_t len,
                                   size_t* out_len) {
     const unsigned char* q = (const unsigned char*)p;
     const yep_text_kernels* k = yep_text_active();
     size_t i = 0;
-    int found = 0;
+    int exotic = 0;
     while (i < len) {
         ptrdiff_t hit = k->stopset_find(&yep_norm_break_leads, (const char*)q + i, len - i);
         if (hit < 0) {
             break;
         }
         i += (size_t)hit;
-        if (q[i] == 0xC2 && i + 1 < len && q[i + 1] == 0x85) {
-            found = 1;
-            break;
-        }
-        if (q[i] == 0xE2 && i + 2 < len && q[i + 1] == 0x80 &&
-            (q[i + 2] == 0xA8 || q[i + 2] == 0xA9)) {
+        exotic = 1; /* any NEL/LS/PS lead byte needs the context walk */
+        break;
+    }
+    if (!exotic) {
+        return NULL;
+    }
+    yep_norm_ctx probe;
+    memset(&probe, 0, sizeof(probe));
+    probe.tok_start = 1;
+    int found = 0;
+    for (size_t j = 0; j < len;) {
+        yep_norm_tok t = norm_step(&probe, p, len, j);
+        if (t.fold_off >= 0) {
             found = 1;
             break;
         }
-        i++;
+        j += t.n;
     }
     if (!found) {
-        return NULL;
+        return NULL; /* the exotic bytes all ride inside quoted regions:
+                      * the zero-copy path keeps them raw (the fix) */
     }
     char* out = yep_alloc(sys, len + 1);
     if (out == NULL) {
         return NULL;
     }
+    yep_norm_ctx cp;
+    memset(&cp, 0, sizeof(cp));
+    cp.tok_start = 1;
     size_t o = 0;
     for (size_t j = 0; j < len;) {
-        if (q[j] == 0xC2 && j + 1 < len && q[j + 1] == 0x85) {
-            out[o++] = '\n';
-            j += 2;
-        } else if (q[j] == 0xE2 && j + 2 < len && q[j + 1] == 0x80 &&
-                   (q[j + 2] == 0xA8 || q[j + 2] == 0xA9)) {
-            out[o++] = '\n';
-            j += 3;
+        yep_norm_tok t = norm_step(&cp, p, len, j);
+        if (t.fold_off < 0) {
+            memcpy(out + o, p + j, t.n);
+            o += t.n;
         } else {
-            out[o++] = p[j++];
+            memcpy(out + o, p + j, (size_t)t.fold_off);
+            o += (size_t)t.fold_off;
+            out[o++] = '\n';
+            memcpy(out + o, p + j + t.fold_off + t.fold_len,
+                   t.n - (size_t)t.fold_off - (size_t)t.fold_len);
+            o += t.n - (size_t)t.fold_off - (size_t)t.fold_len;
         }
+        j += t.n;
     }
     out[o] = '\0';
     if (out_len != NULL) {
@@ -4101,6 +4510,48 @@ void yep_engine_destroy(yep_engine* e) {
 /* A quote opens a scalar only where a value can begin; anywhere else
  * it is literal plain text (the apostrophe in "don't"). This is exact
  * on valid YAML: a quote at a value position always opens. */
+/* A line-start '|' / '>' is a block-scalar header only when the rest
+ * of the line fits the grammar: optional chomping (+/-) and indent
+ * (1-9) indicators in either order, then spaces/tabs, then an optional
+ * space-led comment, then EOL. "| : [" is a plain mapping key whose
+ * value opens a flow — treating it as a header skipped the '[' and let
+ * a '---' cut land inside the still-open flow (#645: whole buffered
+ * that flow to the depth cap, chunked cut and errored at 2:1).
+ * Returns 1 valid, 0 definitely not a header, -1 truncated (the tail
+ * may still complete a valid header). */
+static int step_bs_header(const char* p, size_t len, size_t i) {
+    size_t k = i + 1;
+    int chomp = 0, indent = 0, space = 0;
+    while (k < len) {
+        char c = p[k];
+        if (c == '+' || c == '-') {
+            if (chomp || space)
+                return 0;
+            chomp = 1;
+            k++;
+            continue;
+        }
+        if (c >= '1' && c <= '9') {
+            if (indent || space)
+                return 0;
+            indent = 1;
+            k++;
+            continue;
+        }
+        if (c == ' ' || c == '\t') {
+            space = 1;
+            k++;
+            continue;
+        }
+        if (c == '\n' || c == '\r')
+            return 1;
+        if (c == '#' && space)
+            return 1; /* a comment needs separation */
+        return 0;
+    }
+    return -1; /* the chunk ended mid-line */
+}
+
 /* Advances quote/flow/comment state over buf[scanned, len) and returns
  * the offset of the LAST safe cut (0 = none). When the final bytes'
  * meaning depends on bytes not yet fed (a quote that may be doubled,
@@ -4386,24 +4837,33 @@ static size_t step_scan(yep_stepstate* st) {
                 (i + 1 >= st->len || p[i + 1] == ' ' || p[i + 1] == '\t' || p[i + 1] == '\n' ||
                  p[i + 1] == '\r' || (p[i + 1] >= '0' && p[i + 1] <= '9') || p[i + 1] == '+' ||
                  p[i + 1] == '-')) {
-                uint32_t ind = 0;
-                size_t k = i;
-                while (k > 0 && p[k - 1] == ' ') {
-                    k--;
-                    ind++;
-                }
-                st->bs_open = 1;
-                st->bs_indent = ind;
-                size_t e = i;
-                while (e < st->len && p[e] != '\n') {
-                    e++;
-                }
-                if (e >= st->len) {
-                    hold = i; /* an incomplete header line */
+                int hdr = step_bs_header(p, st->len, i);
+                if (hdr < 0) {
+                    hold = i; /* the header line may still complete */
                     break;
                 }
-                i = e;
-                continue;
+                if (hdr > 0) {
+                    uint32_t ind = 0;
+                    size_t k = i;
+                    while (k > 0 && p[k - 1] == ' ') {
+                        k--;
+                        ind++;
+                    }
+                    st->bs_open = 1;
+                    st->bs_indent = ind;
+                    size_t e = i;
+                    while (e < st->len && p[e] != '\n') {
+                        e++;
+                    }
+                    if (e >= st->len) {
+                        hold = i; /* an incomplete header line */
+                        break;
+                    }
+                    i = e;
+                    continue;
+                }
+                /* Not a header ("| : [" is a mapping key): fall through
+                 * so the line's flow/quote bytes are counted. */
             }
             if (c == '-') {
                 if (i + 3 >= st->len) {
@@ -4660,9 +5120,23 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
                     p0++;
                 }
                 size_t p1 = p0;
-                while (p1 < li.end && e->p[p1] != ' ' && e->p[p1] != '\t' && e->p[p1] != '#') {
+                while (p1 < li.end && e->p[p1] != ' ' && e->p[p1] != '\t') {
                     p1++;
                 }
+                {
+                    /* a '#' inside the prefix token is NOT a comment:
+                     * libyaml wants whitespace before a comment and
+                     * rejects the glued form ("did not find expected
+                     * whitespace or line break while scanning a %TAG
+                     * directive"). Treating it as a comment registered
+                     * a bogus prefix and split the whole/chunked
+                     * verdicts (catch 154). */
+                    const char* hash = (const char*)memchr(e->p + p0, '#', p1 - p0);
+                    if (hash != NULL) {
+                        e_fail(e, YEP_ERR_BAD_DIRECTIVE, (size_t)(hash - e->p));
+                        return -1;
+                    }
+                }
                 {
                     /* libyaml: only blanks and an optional comment may
                      * follow the prefix ("did not find expected comment
@@ -4678,6 +5152,36 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
                         return -1;
                     }
                 }
+                {
+                    /* libyaml validates the handle: "!", "!!", or
+                     * "!word!" (word = alnum/-), and a prefix is
+                     * required ("did not find expected whitespace
+                     * while scanning a %TAG directive"). The lenient
+                     * accept registered "!'" and let the whole parse
+                     * succeed where the chunked one rejected
+                     * (catch 154). */
+                    int handle_ok = 0;
+                    size_t hl = h1 - h0;
+                    if (hl == 1 && e->p[h0] == '!') {
+                        handle_ok = 1;
+                    } else if (hl == 2 && e->p[h0] == '!' && e->p[h0 + 1] == '!') {
+                        handle_ok = 1;
+                    } else if (hl >= 3 && e->p[h0] == '!' && e->p[h1 - 1] == '!') {
+                        handle_ok = 1;
+                        for (size_t k = h0 + 1; k + 1 < h1; k++) {
+                            if (!(yep_ct_is((unsigned char)e->p[k], YEP_CT_ALPHA) ||
+                                  yep_ct_is((unsigned char)e->p[k], YEP_CT_DIGIT) ||
+                                  e->p[k] == '-')) {
+                                handle_ok = 0;
+                                break;
+                            }
+                        }
+                    }
+                    if (!handle_ok || p1 == p0) {
+                        e_fail(e, YEP_ERR_BAD_DIRECTIVE, h0);
+                        return -1;
+                    }
+                }
                 if (h1 > h0 && p1 > p0 && e->p[h0] == '!') {
                     e->tagmap[e->tagmap_n].handle.p = e->p + h0;
                     e->tagmap[e->tagmap_n].handle.len = (uint32_t)(h1 - h0);
@@ -4690,6 +5194,7 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
                 e->doc_begin = li.offset; /* libyaml: DOCUMENT_START's
                                            * start_mark includes the
                                            * leading directive lines */
+                e->doc_begin_line = e->line;
             }
             e_line_done(e, li.end);
             continue;
@@ -4734,10 +5239,19 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
             }
             e_event_init(&ev, YEP_EV_DOCUMENT_START);
             ev.style = 1; /* explicit marker */
-            e_pos_mark(e, e->doc_begin != (size_t)-1 ? e->doc_begin : li.offset + li.indent,
-                       &ev.line, &ev.col);
+            if (e->doc_begin != (size_t)-1) {
+                e_pos_mark(e, e->doc_begin, &ev.line, &ev.col);
+            } else if (e->doc_begin_line != 0) {
+                /* the directive line was consumed by an EARLIER step
+                 * run: its offset is gone, the absolute line is not */
+                ev.line = e->doc_begin_line;
+                ev.col = 1;
+            } else {
+                e_pos_mark(e, li.offset + li.indent, &ev.line, &ev.col);
+            }
             e_pos_mark(e, li.offset + li.indent + 3, &ev.end_line, &ev.end_col);
             e->doc_begin = (size_t)-1;
+            e->doc_begin_line = 0;
             if (emit_now(e, &ev) != 0) {
                 return -2;
             }
@@ -4816,6 +5330,7 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
                 e->doc_content = 0;
                 e->saw_yaml = 0;
                 e->doc_begin = (size_t)-1;
+                e->doc_begin_line = 0;
             }
             e_line_done(e, li.end);
             continue;
@@ -4825,12 +5340,16 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
             e_event_init(&ev, YEP_EV_DOCUMENT_START);
             if (e->doc_begin != (size_t)-1) {
                 e_pos_mark(e, e->doc_begin, &ev.line, &ev.col);
+            } else if (e->doc_begin_line != 0) {
+                ev.line = e->doc_begin_line; /* a directive split across runs */
+                ev.col = 1;
             } else {
                 e_pos_mark(e, li.offset + li.indent, &ev.line, &ev.col);
             }
             ev.end_line = ev.line;
             ev.end_col = ev.col;
             e->doc_begin = (size_t)-1;
+            e->doc_begin_line = 0;
             if (emit_now(e, &ev) != 0) {
                 return -2;
             }