yeptris 0.6.31.2-arm-linux-eabihf → 0.6.33.1-arm-linux-eabihf

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,7 @@
20
20
  #include "common/chartype.h"
21
21
  #include "common/nametab.h"
22
22
  #include "common/simd_text.h"
23
+ #include "encoding/encoding.h"
23
24
  #include "engine.h"
24
25
  #include "memory/allocator.h"
25
26
  #include "memory/pool.h"
@@ -75,8 +76,19 @@ struct yep_engine {
75
76
  yep_frame frames[YEP_MAX_DEPTH];
76
77
  int depth;
77
78
 
79
+ int front_done; /* the encoding front-end ran on this engine:
80
+ * parse/values pre-strip and set it; later
81
+ * chunked runs never re-sniff (mid-stream
82
+ * U+FEFF is content, not a mark) */
78
83
  yep_nametab anchors; /* name interning: O(1) define/lookup at any scale */
79
84
  uint32_t anchor_serial; /* 1-based ordinals; the value IS the id */
85
+ uint8_t* anchor_placed; /* per-ordinal: the anchor's NODE has been
86
+ * placed in the output (an alias to an
87
+ * unplaced anchor would precede its
88
+ * definition and the re-parse would reject
89
+ * the document — the nightly's run
90
+ * 38541356075) */
91
+ uint32_t anchor_placed_cap;
80
92
  /* repeat-alias memo (TODO.restructure/71): offset/len/ordinal of
81
93
  * the last alias resolved — hit by memcmp, not by the table */
82
94
  uint32_t alias_memo_at;
@@ -94,9 +106,23 @@ struct yep_engine {
94
106
  yep_view pend_anchor, pend_tag;
95
107
  uint32_t pend_anchor_id; /* the pend anchor's ordinal */
96
108
 
97
- int doc_content; /* any node emitted for the current document */
98
- int q_key_pending; /* an explicit '?' key awaits its ':' value line */
99
- int q_map_depth; /* frame depth of the '?' mapping (flush guard) */
109
+ int doc_content; /* any node emitted for the current document */
110
+ int parsing_final; /* engine_run_impl's emit_end: the buffer is the
111
+ * complete input (pre-scan errors are real) vs a
112
+ * partial step feed (an unclosed flow is
113
+ * incomplete, not malformed) */
114
+ int q_key_pending; /* an explicit '?' key awaits its ':' value line */
115
+ uint32_t cv_line; /* the line whose colon-value is being parsed: a
116
+ * key indicator on it sits where a value belongs
117
+ * (": ?") — libyaml rejects */
118
+ int qmd_stack[16]; /* the '?' pending pairs' map depths — a STACK:
119
+ * nested '? ?' pairs stack two pendings and a
120
+ * scalar slot let the inner clobber the
121
+ * outer's level (the nightly's run 38387614146) */
122
+ uint32_t qmd_line[16]; /* each pending's open line: a LATER-line '?'
123
+ * at the same level completes the previous
124
+ * pair (libyaml's same-indent KEY rule) */
125
+ int qmd_n;
100
126
  int q_value_pending; /* an explicit key was emitted; its value is null
101
127
  until a ':' line supplies one */
102
128
  /* %TAG handle map for the current document (engine = tag SSOT) */
@@ -115,27 +141,30 @@ struct yep_engine {
115
141
  * the key's events, but ':' is only seen after them). */
116
142
  yep_event* ev_buf;
117
143
  uint32_t ev_buf_n, ev_buf_cap;
118
- uint32_t ev_scopes; /* active entry-buffer scopes */
119
- char* norm_buf; /* owned input copy with exotic breaks normalized */
120
- const struct yep_resolver* resolver; /* implicit-typing schema (10) */
121
- int max_depth; /* runtime nesting limit (options) */
122
- uint16_t rec; /* e_node<->e_parse_value call depth:
123
- the props/value-line chain opens NO
124
- container frame, so e->depth cannot
125
- bound it (the nightly's stack
126
- overflow crasher) */
127
- int last_root_flow; /* a root-level flow node just completed */
128
- int tag_undef; /* an unresolved !handle! was seen (QLJ7) */
129
- int doc_inline; /* content node opened on the --- line */
130
- uint16_t flow_floor; /* parent column of the current block-level flow */
131
- int flow_enforce; /* flow continuation lines must out-indent it */
132
- int compat_grammar; /* 11-COMPAT: libyaml/psych grammar parity —
133
- * the flow indent floor (9C9N/VJP3) is a
134
- * yaml-test-suite rule libyaml never
135
- * enforced, and real-world locale files
136
- * close multi-line flows at the parent's
137
- * column; the floor stays on for the
138
- * strict 1.2 engine the suite runs */
144
+ uint32_t ev_scopes; /* active entry-buffer scopes */
145
+ char* norm_buf;
146
+ unsigned char* enc_buf;
147
+ /* owned UTF-16/32 transcode scratch (run-owned) */ /* owned input copy with exotic breaks
148
+ normalized */
149
+ const struct yep_resolver* resolver; /* implicit-typing schema (10) */
150
+ int max_depth; /* runtime nesting limit (options) */
151
+ uint16_t rec; /* e_node<->e_parse_value call depth:
152
+ the props/value-line chain opens NO
153
+ container frame, so e->depth cannot
154
+ bound it (the nightly's stack
155
+ overflow crasher) */
156
+ int last_root_flow; /* a root-level flow node just completed */
157
+ int tag_undef; /* an unresolved !handle! was seen (QLJ7) */
158
+ int doc_inline; /* content node opened on the --- line */
159
+ uint16_t flow_floor; /* parent column of the current block-level flow */
160
+ int flow_enforce; /* flow continuation lines must out-indent it */
161
+ int compat_grammar; /* 11-COMPAT: libyaml/psych grammar parity —
162
+ * the flow indent floor (9C9N/VJP3) is a
163
+ * yaml-test-suite rule libyaml never
164
+ * enforced, and real-world locale files
165
+ * close multi-line flows at the parent's
166
+ * column; the floor stays on for the
167
+ * strict 1.2 engine the suite runs */
139
168
  };
140
169
 
141
170
  /* ------------------------------------------------------------ forwards */
@@ -167,7 +196,7 @@ static void e_pos_mark(const yep_engine* e, size_t at, uint32_t* line, uint32_t*
167
196
  * walk below it, or ls never decreases (the infinite-loop bug
168
197
  * the Stepping.Directive hang found) */
169
198
  size_t q = ls - 1;
170
- while (q > 0 && e->p[q - 1] != '\n') {
199
+ while (q > 0 && e->p[q - 1] != '\n' && e->p[q - 1] != '\r') {
171
200
  q--;
172
201
  }
173
202
  ls = q; /* line_start(l-1) */
@@ -197,7 +226,10 @@ static void e_event_init(yep_event* ev, yep_event_type t) {
197
226
  ev->type = t;
198
227
  }
199
228
 
229
+ static void e_anchor_mark(yep_engine* e, uint32_t id);
230
+
200
231
  static int emit_now(yep_engine* e, const yep_event* ev) {
232
+ e_anchor_mark(e, ev->anchor_id);
201
233
  if (ev->type == YEP_EV_SCALAR) {
202
234
  /* Resolution happens exactly here: every scalar path flows
203
235
  * through this choke point (the resolver is the typing SSOT) */
@@ -382,11 +414,37 @@ static uint32_t anchor_id_of(yep_engine* e, yep_view name) {
382
414
  return id;
383
415
  }
384
416
 
417
+ /* The anchor's node has been PLACED (an event emitted, or the node
418
+ * handed to a fused sink callback): aliases to it may parse. */
419
+ static void e_anchor_mark(yep_engine* e, uint32_t id) {
420
+ if (id != 0 && id <= e->anchor_serial && e->anchor_placed != NULL) {
421
+ e->anchor_placed[id - 1] = 1;
422
+ }
423
+ }
424
+
385
425
  static uint32_t anchor_define(yep_engine* e, yep_view name) {
386
426
  /* Last definition wins (libyaml parity); OOM leaves the anchor
387
427
  * undefined so its alias errors. The VALUE is the anchor's 1-based
388
428
  * ordinal — sinks key on it and skip the name hash entirely. */
389
429
  uint32_t id = ++e->anchor_serial;
430
+ if (id > e->anchor_placed_cap) {
431
+ uint32_t cap = e->anchor_placed_cap ? e->anchor_placed_cap : 64;
432
+ while (cap < id) {
433
+ cap *= 2;
434
+ }
435
+ uint8_t* nb = yep_alloc(e->sys, cap);
436
+ if (nb == NULL) {
437
+ e->anchor_serial--;
438
+ return 0; /* OOM: the anchor stays undefined, its alias errors */
439
+ }
440
+ if (e->anchor_placed != NULL) {
441
+ memcpy(nb, e->anchor_placed, e->anchor_placed_cap);
442
+ yep_free(e->sys, e->anchor_placed);
443
+ }
444
+ memset(nb + e->anchor_placed_cap, 0, cap - e->anchor_placed_cap);
445
+ e->anchor_placed = nb;
446
+ e->anchor_placed_cap = cap;
447
+ }
390
448
  yep_nametab_set(&e->anchors, name, id);
391
449
  return id;
392
450
  }
@@ -465,13 +523,18 @@ static int e_open_map(yep_engine* e, uint16_t col, uint32_t line, uint32_t coln,
465
523
 
466
524
  /* Emits the null value for an explicit key that never got its ':' line. */
467
525
  static int e_flush_q_value(yep_engine* e) {
468
- if (!e->q_value_pending) {
526
+ if (!e->q_value_pending || e->qmd_n == 0) {
469
527
  return 0;
470
528
  }
471
- if (e->depth > e->q_map_depth) {
472
- return 0; /* the key's own structure is still open */
529
+ int top = e->qmd_stack[e->qmd_n - 1];
530
+ if (e->depth != top) {
531
+ return 0; /* the pending pair's map is not the current level */
473
532
  }
474
- e->q_value_pending = 0;
533
+ e->qmd_n--;
534
+ /* nested '?' pairs stack one pending per level — the flag stays up
535
+ * while the stack still holds one, so the outer null flushes at its
536
+ * own level (the nightly's run 38387614146: `? ? k : a` lost it) */
537
+ e->q_value_pending = (e->qmd_n > 0);
475
538
  yep_event ev;
476
539
  e_event_init(&ev, YEP_EV_SCALAR);
477
540
  ev.style = YEP_STYLE_PLAIN;
@@ -479,6 +542,20 @@ static int e_flush_q_value(yep_engine* e) {
479
542
  return emit_now(e, &ev) == 0 ? 0 : -2;
480
543
  }
481
544
 
545
+ /* The pair-value completion: every `key: value` arm funnels its value
546
+ * parse through here. Under an AFTER_Q parse (a nested '?' pair whose
547
+ * value arrived inline) the innermost '?' pending resolves — without
548
+ * this pop the stack keeps a stale entry and the outer pair's null
549
+ * flushes inside the wrong map (the nightly's run 38387614146). */
550
+ static int e_pair_value_done(yep_engine* e, yep_ctx ctx, uint16_t key_col) {
551
+ int rc = e_parse_value(e, YEP_CTX_AFTER_COLON, key_col);
552
+ if (rc >= 0 && ctx == YEP_CTX_AFTER_Q && e->qmd_n > 0 &&
553
+ e->depth == e->qmd_stack[e->qmd_n - 1]) {
554
+ e->qmd_n--; /* the inline value resolved the innermost '?' pending */
555
+ }
556
+ return rc;
557
+ }
558
+
482
559
  static int e_close_to(yep_engine* e, int to_depth) {
483
560
  while (e->depth > to_depth) {
484
561
  /* the pending '?' value flushes once the key's structure closed */
@@ -486,7 +563,7 @@ static int e_close_to(yep_engine* e, int to_depth) {
486
563
  if (rc != 0) {
487
564
  return rc;
488
565
  }
489
- if (e->q_value_pending && e->depth == e->q_map_depth) {
566
+ if (e->q_value_pending && e->qmd_n > 0 && e->depth == e->qmd_stack[e->qmd_n - 1]) {
490
567
  continue; /* flushed the null; this frame closes next round */
491
568
  }
492
569
  e->depth--;
@@ -629,6 +706,18 @@ static int e_quoted_floor(yep_engine* e, yep_event* ev, uint16_t min_indent, int
629
706
  return e_fail(e, YEP_ERR_INVALID_ESCAPE, i); /* lone low */
630
707
  }
631
708
  }
709
+ if (esc == 'U') {
710
+ /* 8 hex digits reach far past U+10FFFF — the nightly's
711
+ * "\UA66DA66D" crasher: parse accepted an unencodable
712
+ * code point and the emitted bytes never re-parsed */
713
+ uint32_t cp = 0;
714
+ for (size_t k = 1; k <= 8; k++) {
715
+ cp = (cp << 4) | (uint32_t)hexval((unsigned char)e->p[i + 1 + k]);
716
+ }
717
+ if (cp > 0x10FFFF) {
718
+ return e_fail(e, YEP_ERR_INVALID_ESCAPE, i);
719
+ }
720
+ }
632
721
  i += 1 + need; /* the digits are not escapes themselves */
633
722
  continue;
634
723
  }
@@ -709,9 +798,12 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
709
798
  e->pos++;
710
799
  int chomp = 0;
711
800
  int explicit_indent = 0;
801
+ int saw_digit = 0;
802
+ int saw_blank = 0;
712
803
  while (e->pos < e->len && e->p[e->pos] != '\n' && e->p[e->pos] != '\r') {
713
804
  char c = e->p[e->pos];
714
805
  if (c == ' ' || c == '\t') {
806
+ saw_blank = 1;
715
807
  e->pos++;
716
808
  continue;
717
809
  }
@@ -719,12 +811,22 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
719
811
  e_skip_to_eol(e);
720
812
  break;
721
813
  }
814
+ if (saw_blank || (c == '-' && chomp != 0) || (c == '+' && chomp != 0) ||
815
+ (c >= '1' && c <= '9' && saw_digit)) {
816
+ /* libyaml: the params ride the indicator DIRECTLY (a
817
+ * spaced "| 2" is trailing content, not a param — "did
818
+ * not find expected comment or line break"), one
819
+ * indentation digit and one chomp sign at most (the
820
+ * nightly's run 38119775008 family) */
821
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
822
+ }
722
823
  if (c == '-') {
723
824
  chomp = 1;
724
825
  } else if (c == '+') {
725
826
  chomp = 2;
726
827
  } else if (c >= '1' && c <= '9') {
727
828
  explicit_indent = c - '0';
829
+ saw_digit = 1;
728
830
  } else {
729
831
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
730
832
  }
@@ -748,9 +850,21 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
748
850
  e->pos == e->line_start ? e_line_info_here(e) : yep_scan_line(e->p, e->len, e->pos);
749
851
  int blank = (li.flags & YEP_LF_BLANK) != 0;
750
852
  if (!blank) {
751
- if (saw_content && parent_col < 0 &&
752
- (li.flags & (YEP_LF_DOC_START | YEP_LF_DOC_END | YEP_LF_DIRECTIVE))) {
753
- break; /* root block ends at ---/.../% even at the content column */
853
+ if (parent_col < 0 && (li.flags & (YEP_LF_DOC_START | YEP_LF_DOC_END))) {
854
+ /* col-0 ---/... are document boundaries ALWAYS — even
855
+ * with no content yet: libyaml scans the marker as a
856
+ * document token regardless, so '>\n--- ' is an EMPTY
857
+ * block scalar plus a second (empty) document, not a
858
+ * block whose content is the marker line (the nightly's
859
+ * run 38544774709: whole saw one doc, the chunked cut
860
+ * at the marker saw two) */
861
+ break;
862
+ }
863
+ if (saw_content && parent_col < 0 && (li.flags & YEP_LF_DIRECTIVE)) {
864
+ /* a %-line after content ends the root block; as the
865
+ * FIRST line it is content (M7A3/W4TN: the block's
866
+ * value is '%!PS-Adobe-2.0 ...') */
867
+ break;
754
868
  }
755
869
  if (content_indent < 0) {
756
870
  content_indent = li.indent;
@@ -867,6 +981,14 @@ static int e_block_scalar(yep_engine* e, yep_event* ev, int parent_col) {
867
981
  }
868
982
  }
869
983
  trailing_breaks += pending_breaks;
984
+ if (!saw_content && trailing_breaks > 0 && e->pos == e->len && e->len > 0 &&
985
+ e->p[e->len - 1] != '\n' && e->p[e->len - 1] != '\r') {
986
+ /* keep-chomping counts COMPLETE blank lines only: the final
987
+ * blank line carries no break at EOF, so there is nothing to
988
+ * keep ("|+\n " is "" where the pre-fix kept "\n"; two
989
+ * complete blanks stay "\n\n" -- libyaml/psych oracle) */
990
+ trailing_breaks--;
991
+ }
870
992
  if (saw_content ? (max_blank_indent > content_indent) : blank_tab) {
871
993
  /* a leading blank line out-indents the first content line; a
872
994
  * tab-only line before any content is an error (5LLU/Y79Y) */
@@ -938,6 +1060,15 @@ static int e_plain_multiline(yep_engine* e, yep_span s0, uint32_t block_floor, y
938
1060
  while (cs < e->len && (e->p[cs] == ' ' || e->p[cs] == '\t')) {
939
1061
  cs++; /* continuation leading white space is stripped */
940
1062
  }
1063
+ if (cs < e->len && e->p[cs] == '#') {
1064
+ /* a '#' opening a continuation line is a comment: the
1065
+ * plain ends here and nothing folds -- the scan below
1066
+ * returned an empty piece that still took a fold slot,
1067
+ * adding a trailing fold-space ("a\n#" parsed as "a ",
1068
+ * libyaml "a": the nightly's combined-build round) */
1069
+ e_skip_to_eol(e);
1070
+ break;
1071
+ }
941
1072
  yep_span s = yep_scan_plain(e->p, e->len, cs, 0);
942
1073
  if (s.term == YEP_TERM_COLON) {
943
1074
  return e_fail(e, YEP_ERR_UNEXPECTED, s.end);
@@ -994,6 +1125,17 @@ static int e_hex_val(unsigned char c) {
994
1125
  /* Tags carry URI escapes (%XX) in every form (verbatim, !!, %TAG). */
995
1126
  static yep_view e_tag_uri_decode(yep_engine* e, yep_view v) {
996
1127
  const char* p = v.p;
1128
+ /* every '%' must be followed by two hex digits — libyaml rejects
1129
+ * the family ("did not find URI escaped octet"); a stray '%' left
1130
+ * raw would decode differently on re-parse of the re-encoded form
1131
+ * (the nightly's tag-escape byte-instability) */
1132
+ for (uint32_t i = 0; i < v.len; i++) {
1133
+ if (p[i] == '%' && (i + 2 >= v.len || e_hex_val((unsigned char)p[i + 1]) < 0 ||
1134
+ e_hex_val((unsigned char)p[i + 2]) < 0)) {
1135
+ e->tag_undef = 1; /* e_props reports it */
1136
+ return v;
1137
+ }
1138
+ }
997
1139
  for (uint32_t i = 0; i + 2 < v.len; i++) {
998
1140
  if (p[i] == '%' && e_hex_val((unsigned char)p[i + 1]) >= 0 &&
999
1141
  e_hex_val((unsigned char)p[i + 2]) >= 0) {
@@ -1103,6 +1245,17 @@ static void e_props(yep_engine* e, yep_view* anchor, yep_view* tag) {
1103
1245
  e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1104
1246
  return;
1105
1247
  }
1248
+ if (e->pos < e->len && e->p[e->pos] == '#') {
1249
+ /* the name ends at a blank or flow indicator; a glued
1250
+ * '#' is junk on the name (libyaml: "did not find
1251
+ * expected alphabetic or numeric character") — the
1252
+ * nightly's run 38121449804, "&f#o": this scan kept
1253
+ * the anchor and DROPPED the bytes after '#',
1254
+ * mangling the node while the runner kept a different
1255
+ * shape */
1256
+ e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1257
+ return;
1258
+ }
1106
1259
  } else if (c == '!') {
1107
1260
  size_t start = e->pos;
1108
1261
  e->pos++;
@@ -1122,10 +1275,22 @@ static void e_props(yep_engine* e, yep_view* anchor, yep_view* tag) {
1122
1275
  }
1123
1276
  e->pos = gt + 1;
1124
1277
  } else {
1125
- while (e->pos < e->len && yep_scan_prop_char((unsigned char)e->p[e->pos])) {
1278
+ /* libyaml scans URI chars only: bytes like '>' or '|'
1279
+ * end the suffix and surface as the follower, which
1280
+ * the check below rejects */
1281
+ while (e->pos < e->len && yep_scan_tag_char((unsigned char)e->p[e->pos])) {
1126
1282
  e->pos++;
1127
1283
  }
1128
1284
  }
1285
+ if (e->pos < e->len) {
1286
+ unsigned char after = (unsigned char)e->p[e->pos];
1287
+ if (!(after == ' ' || after == '\t' || after == '\n' || after == '\r' ||
1288
+ after == ',' || after == '[' || after == ']' || after == '{' ||
1289
+ after == '}')) {
1290
+ e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1291
+ return;
1292
+ }
1293
+ }
1129
1294
  yep_view raw = {e->p + start, (uint32_t)(e->pos - start)};
1130
1295
  *tag = e_resolve_tag(e, raw);
1131
1296
  if (e->tag_undef) {
@@ -1162,10 +1327,31 @@ static int e_alias(yep_engine* e, yep_event* ev) {
1162
1327
  return e_fail(e, YEP_ERR_UNEXPECTED, star);
1163
1328
  }
1164
1329
  uint32_t target = anchor_id_of(e, name);
1165
- if (target == 0) {
1330
+ if (target == 0 || (target <= e->anchor_serial && e->anchor_placed != NULL &&
1331
+ e->anchor_placed[target - 1] == 0)) {
1332
+ /* undefined, or defined-but-not-yet-placed: the props scan
1333
+ * registers the name before the anchor's node reaches the
1334
+ * output, so an alias here would precede its definition and
1335
+ * the re-parse would reject the document (libyaml: "found
1336
+ * undefined alias") */
1166
1337
  e->pos = star;
1167
1338
  return e_fail(e, YEP_ERR_UNDEFINED_ALIAS, star);
1168
1339
  }
1340
+ if (e->pos < e->len && e->p[e->pos] == '#') {
1341
+ /* the name ends at a blank or flow indicator; a glued '#' is
1342
+ * junk (the anchor-name follower law, catch 43's family) */
1343
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1344
+ }
1345
+ e_skip_inline_space(e);
1346
+ if (e->pos < e->len && (e->p[e->pos] == '&' || e->p[e->pos] == '!' || e->p[e->pos] == '*')) {
1347
+ /* an alias node is COMPLETE — it carries no props and nothing
1348
+ * can follow it on the line but a separator: "<<: *base &bar"
1349
+ * (the nightly's run 38124486657 family) errored in libyaml
1350
+ * ("did not find expected key") while the DOM route absorbed
1351
+ * the stray props and the value drain rejected — the routes
1352
+ * split */
1353
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1354
+ }
1169
1355
  ev->type = YEP_EV_ALIAS;
1170
1356
  ev->value = name;
1171
1357
  ev->borrowed = 1; /* the name is an input view (TODO.restructure/73):
@@ -1173,7 +1359,6 @@ static int e_alias(yep_engine* e, yep_event* ev) {
1173
1359
  arena copy per alias */
1174
1360
  ev->anchor_id = target;
1175
1361
  e_pos_mark(e, e->pos, &ev->end_line, &ev->end_col);
1176
- e_skip_inline_space(e);
1177
1362
  return 0;
1178
1363
  }
1179
1364
 
@@ -1220,16 +1405,51 @@ static int e_skip_flow(yep_engine* e) {
1220
1405
  int n = 0;
1221
1406
  stk[n++] = (e->p[e->pos] == '[') ? 0 : 1;
1222
1407
  e->pos++;
1408
+ /* quotes open only at a TOKEN start: a quote byte inside a tag
1409
+ * suffix ("[!'str b]") or a plain ("[a'b]") is content — libyaml
1410
+ * accepts both; the pre-scan must not read them as a quoted
1411
+ * scalar (the nightly's run 38171171360) */
1412
+ int tok_start = 1;
1223
1413
  while (e->pos < e->len && n > 0) {
1224
1414
  unsigned char c = (unsigned char)e->p[e->pos];
1225
- if (c == '"' || c == '\'') {
1415
+ if (tok_start && (c == '!' || c == '&' || c == '*')) {
1416
+ /* props/aliases ride the span: skip them the way the real
1417
+ * scan does — a verbatim tag to its '>', names over the
1418
+ * tag-char run (no quote/bracket bytes in the run) */
1419
+ e->pos++;
1420
+ if (c == '!' && e->pos < e->len && e->p[e->pos] == '<') {
1421
+ while (e->pos < e->len && e->p[e->pos] != '>' && e->p[e->pos] != '\n' &&
1422
+ e->p[e->pos] != '\r' && e->p[e->pos] != ' ' && e->p[e->pos] != '\t') {
1423
+ e->pos++;
1424
+ }
1425
+ if (e->pos < e->len && e->p[e->pos] == '>') {
1426
+ e->pos++;
1427
+ }
1428
+ tok_start = 0;
1429
+ continue;
1430
+ }
1431
+ while (e->pos < e->len && yep_scan_tag_char((unsigned char)e->p[e->pos])) {
1432
+ e->pos++;
1433
+ }
1434
+ tok_start = 0;
1435
+ continue;
1436
+ }
1437
+ if (tok_start && (c == '"' || c == '\'')) {
1226
1438
  int esc2 = 0;
1227
1439
  const yep_text_kernels* k = yep_text_active();
1228
1440
  ptrdiff_t r = k->quote_scan(e->p + e->pos + 1, e->len - e->pos - 1, (char)c, &esc2);
1229
1441
  if (r < 0) {
1230
- return e_fail(e, YEP_ERR_UNTERMINATED_QUOTE, e->pos);
1442
+ /* an unterminated quote inside the span: NOT-A-KEY — the
1443
+ * real parse (e_flow) owns the error AFTER it has streamed
1444
+ * the flow's open events. The pre-scan's own error here
1445
+ * fired BEFORE those events, so the whole buffer's stream
1446
+ * lost them while the chunked feed had already emitted
1447
+ * theirs (the nightly's round 106); the unbalanced-span
1448
+ * deferral below defers the same way. */
1449
+ return 0;
1231
1450
  }
1232
1451
  e->pos += 1 + (size_t)r + 1;
1452
+ tok_start = 0;
1233
1453
  continue;
1234
1454
  }
1235
1455
  if (c == '#' && e_comment_at(e)) {
@@ -1238,8 +1458,24 @@ static int e_skip_flow(yep_engine* e) {
1238
1458
  }
1239
1459
  if (c == '\n' || c == '\r') {
1240
1460
  e_line_done(e, e->pos);
1461
+ tok_start = 1;
1241
1462
  continue;
1242
1463
  }
1464
+ if (c == ' ' || c == '\t' || c == '[' || c == '{' || c == ',') {
1465
+ tok_start = 1;
1466
+ } else if (c == ':' && e->pos + 1 < e->len &&
1467
+ (e->p[e->pos + 1] == ' ' || e->p[e->pos + 1] == '\t' ||
1468
+ e->p[e->pos + 1] == '\n' || e->p[e->pos + 1] == '\r' ||
1469
+ e->p[e->pos + 1] == '"' || e->p[e->pos + 1] == '\'' ||
1470
+ yep_ct_is((unsigned char)e->p[e->pos + 1], YEP_CT_FLOW_IND))) {
1471
+ /* a separating ':' only — including directly before a
1472
+ * quoted value: "{\"k\"\t:\"[\"}" is JSON- and YAML-flow-
1473
+ * valid, and treating ':' as mid-token left the value's
1474
+ * quotes unscanned, so its CLOSE read as an unterminated
1475
+ * opener (the nightly's round 103) */
1476
+ } else {
1477
+ tok_start = 0;
1478
+ }
1243
1479
  if (c == '[') {
1244
1480
  if (n >= e->max_depth) {
1245
1481
  return e_fail(e, YEP_ERR_DEPTH, e->pos);
@@ -1251,20 +1487,32 @@ static int e_skip_flow(yep_engine* e) {
1251
1487
  }
1252
1488
  stk[n++] = 1;
1253
1489
  } else if (c == ']') {
1254
- if (stk[n - 1] != 0) {
1255
- return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1490
+ if (n == 0 || stk[n - 1] != 0) {
1491
+ /* a stray close outside any candidate span (or a kind
1492
+ * mismatch) is NOT a flow shape: defer to the general
1493
+ * path -- aborting here discarded the pre-scan's whole
1494
+ * document before a single event emitted, while the
1495
+ * chunked feed had already committed the prefix runs
1496
+ * (the nightly's round 113) */
1497
+ return 0;
1256
1498
  }
1257
1499
  n--;
1258
1500
  } else if (c == '}') {
1259
- if (stk[n - 1] != 1) {
1260
- return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1501
+ if (n == 0 || stk[n - 1] != 1) {
1502
+ return 0;
1261
1503
  }
1262
1504
  n--;
1263
1505
  }
1264
1506
  e->pos++;
1265
1507
  }
1266
1508
  if (n != 0) {
1267
- return e_fail(e, YEP_ERR_UNEXPECTED, e->len);
1509
+ /* incomplete: the real parse (e_flow) owns the deferral in BOTH
1510
+ * modes. The pre-emptive final error (removed here) fired
1511
+ * BEFORE the flow's first events, while the chunked feed had
1512
+ * already streamed MAP_START + the first entry — the two routes'
1513
+ * event streams diverged at the error boundary (the nightly's
1514
+ * round 104). e_flow errors on the unclosed span either way. */
1515
+ return 0;
1268
1516
  }
1269
1517
  return 0;
1270
1518
  }
@@ -1306,6 +1554,14 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1306
1554
  if (e->err.code != YEP_ERR_NONE) {
1307
1555
  return -1; /* e_props failed (empty anchor name, unresolved tag) */
1308
1556
  }
1557
+ if (!yep_view_is_empty(anchor) &&
1558
+ (memchr(anchor.p, '"', anchor.len) != NULL || memchr(anchor.p, '\'', anchor.len) != NULL)) {
1559
+ /* anchors have no quoting: a quote byte in a FLOW anchor name
1560
+ * has no safe spelling (Psych: "did not find expected
1561
+ * alphabetic or numeric character"). Block names ride quotes
1562
+ * (the suite's W5VH) — this law is flow-scoped. */
1563
+ return (int)e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1564
+ }
1309
1565
  ev->anchor = anchor;
1310
1566
  ev->tag = tag;
1311
1567
  if (!yep_view_is_empty(anchor)) {
@@ -1333,7 +1589,15 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1333
1589
  return e_quoted_floor(e, ev, 0, 0) == 0 ? 1 : -1;
1334
1590
  }
1335
1591
  if (c == '*') {
1336
- return e_alias(e, ev) == 0 ? 1 : -1;
1592
+ if (e_alias(e, ev) != 0) {
1593
+ return -1;
1594
+ }
1595
+ if (ev->value.len > 0 && (memchr(ev->value.p, '"', ev->value.len) != NULL ||
1596
+ memchr(ev->value.p, '\'', ev->value.len) != NULL)) {
1597
+ /* the alias twin of the flow anchor-name law above */
1598
+ return (int)e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1599
+ }
1600
+ return 1;
1337
1601
  }
1338
1602
  if (c == '[' || c == '{') {
1339
1603
  return 2;
@@ -1355,6 +1619,7 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1355
1619
  }
1356
1620
  e->fold_n = 0;
1357
1621
  int crossed_break = 0;
1622
+ uint32_t flow_breaks = 1; /* the break count before the next piece */
1358
1623
  for (;;) {
1359
1624
  yep_span s = yep_scan_plain(e->p, e->len, e->pos, 1);
1360
1625
  if (s.end == s.start && e->fold_n == 0) {
@@ -1366,7 +1631,7 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1366
1631
  if (s.end > s.start) {
1367
1632
  e->fold[e->fold_n].content.p = e->p + s.start;
1368
1633
  e->fold[e->fold_n].content.len = s.end - s.start;
1369
- e->fold[e->fold_n].breaks_before = (e->fold_n == 0) ? 0 : 1;
1634
+ e->fold[e->fold_n].breaks_before = (e->fold_n == 0) ? 0 : flow_breaks;
1370
1635
  e->fold_n++;
1371
1636
  }
1372
1637
  if (s.term == YEP_TERM_EOL || s.term == YEP_TERM_COMMENT) {
@@ -1375,21 +1640,58 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1375
1640
  e_skip_to_eol(e);
1376
1641
  break; /* a comment ends the scalar: the next token is new */
1377
1642
  }
1378
- e_line_done(e, e->pos);
1379
1643
  crossed_break = 1;
1380
- {
1644
+ /* Count EVERY break up to the next piece: 1 folds to a
1645
+ * space, n>1 folds to n-1 newlines (the plain law —
1646
+ * "[0\r\r)]" is "0\n)", the nightly's run 39070140324).
1647
+ * The seed is 0 because the loop below consumes the
1648
+ * piece-terminating break itself. Trailing spaces of the
1649
+ * piece's own line are skipped first — yep_scan_plain can
1650
+ * stop before them, and "br == 0 at a space" must never
1651
+ * be read as content (that spun forever on "[0 \n )]"). */
1652
+ uint32_t nbreaks = 0;
1653
+ int ended = 0;
1654
+ for (;;) {
1655
+ e_skip_inline_space(e);
1381
1656
  yep_line_info cl = e_line_info_here(e);
1382
1657
  if (cl.flags & YEP_LF_COMMENT) {
1383
1658
  e_line_done(e, cl.end);
1384
- break; /* a comment line ends the scalar too */
1659
+ ended = 1; /* a comment line ends the scalar too */
1660
+ break;
1661
+ }
1662
+ size_t br = yep_scan_break_len(e->p, e->len, e->pos);
1663
+ if (br == 0) {
1664
+ break; /* content on this line */
1665
+ }
1666
+ nbreaks++;
1667
+ e->pos += br;
1668
+ e->line++;
1669
+ e->line_start = e->pos;
1670
+ e->li_cache_line = 0;
1671
+ e->shape_cache_line = 0;
1672
+ e->facts_cache_line = 0;
1673
+ if (e->pos >= e->len) {
1674
+ ended = 1;
1675
+ break;
1385
1676
  }
1386
1677
  }
1387
- if (e_flow_ws(e) != 0) {
1388
- return -1;
1678
+ flow_breaks = nbreaks;
1679
+ if (ended) {
1680
+ break;
1389
1681
  }
1390
1682
  if (e->pos < e->len && e->p[e->pos] != ']' && e->p[e->pos] != '}' &&
1391
- e->p[e->pos] != ',' && !e_comment_at(e) && e->p[e->pos] != '\n' &&
1392
- e->p[e->pos] != '\r') {
1683
+ e->p[e->pos] != ',' && !e_comment_at(e)) {
1684
+ /* a document marker line ends the scalar (and with it
1685
+ * the unclosed flow) — folding across "---" absorbed
1686
+ * the marker and the whole route's scalar grew past the
1687
+ * chunked route's deferral boundary (the nightly's
1688
+ * round 104) */
1689
+ yep_line_info ml = e_line_info_here(e);
1690
+ if (ml.flags & (YEP_LF_DOC_START | YEP_LF_DOC_END)) {
1691
+ break; /* real document markers end the scalar; a
1692
+ * directive-LIKE '%' line stays content
1693
+ * mid-flow (XLQ9, the UT92 spec example) */
1694
+ }
1393
1695
  continue; /* continuation on a following line */
1394
1696
  }
1395
1697
  break; /* at a boundary; pos is on the separator/close */
@@ -1439,6 +1741,16 @@ static int e_flow_node(yep_engine* e, yep_event* ev, int keyish) {
1439
1741
  static void jx_advance_line(yep_engine* e, size_t* from, size_t to, uint32_t* line,
1440
1742
  size_t* line_start) {
1441
1743
  yep_scan_advance_line(e->p, from, to, line, line_start);
1744
+ /* a lone \r sitting at the walk boundary (to - 1) ends the line
1745
+ * AT to, but the scan's walk cannot peek at p[to] (it does not
1746
+ * know the buffer end) — correct it here with the engine's own
1747
+ * length: the fast path's marks after a \r-containing span sat
1748
+ * one line low while the general kernel marked correctly (the
1749
+ * nightly's runs 38549939326 + 38563005475) */
1750
+ if (to > 0 && to <= e->len && e->p[to - 1] == '\r' && (to == e->len || e->p[to] != '\n')) {
1751
+ (*line)++;
1752
+ *line_start = to;
1753
+ }
1442
1754
  }
1443
1755
 
1444
1756
  /* A flow node followed by ':' on the same line is a KEY: the caller
@@ -1446,7 +1758,11 @@ static void jx_advance_line(yep_engine* e, size_t* from, size_t to, uint32_t* li
1446
1758
  * is_key logic runs. Shared by both fast paths (grammar fact). */
1447
1759
  static int e_flow_key_after(const char* p, size_t len, size_t close) {
1448
1760
  size_t j = close + 1;
1449
- while (j < len && p[j] == ' ') {
1761
+ /* tabs are inline whitespace too: "[]\t\t: |-" defers to the key
1762
+ * machinery like its all-spaces sibling, or the fast path commits
1763
+ * the flow and then errors on the colon it cannot read as a key
1764
+ * separator (the nightly's round 116) */
1765
+ while (j < len && (p[j] == ' ' || p[j] == '\t')) {
1450
1766
  j++;
1451
1767
  }
1452
1768
  return j < len && p[j] == ':' &&
@@ -1455,7 +1771,14 @@ static int e_flow_key_after(const char* p, size_t len, size_t close) {
1455
1771
  }
1456
1772
 
1457
1773
  /* Block-level flow lines must out-indent the parent (9C9N): any line
1458
- * start inside the span at or left of the floor falls back. */
1774
+ * start inside the span at or left of the floor falls back — the
1775
+ * closer's own line included ("a: {\n x: 1\n}\nb: 3" is a suite
1776
+ * error; only libyaml accepts it). The scan must cover the closer
1777
+ * line: its start sits AT the close offset, outside the loop below,
1778
+ * so it is checked explicitly — the general kernel's per-line check
1779
+ * rejects it, and the fast path must agree (the nightly's run
1780
+ * 38280372447: whole vs chunked split across the fast/slow paths and
1781
+ * produced two different verdicts for the same bytes). */
1459
1782
  static int e_flow_floor_ok(const char* p, size_t open, size_t close, uint16_t floor) {
1460
1783
  size_t j = open;
1461
1784
  while (j < close) {
@@ -1472,6 +1795,20 @@ static int e_flow_floor_ok(const char* p, size_t open, size_t close, uint16_t fl
1472
1795
  }
1473
1796
  j++;
1474
1797
  }
1798
+ if (close < (size_t)-1 && close > open) {
1799
+ size_t ls = close;
1800
+ uint32_t ind = 0;
1801
+ while (ls > 0 && p[ls - 1] != '\n') {
1802
+ ls--;
1803
+ }
1804
+ while (ls < close && p[ls] == ' ') {
1805
+ ind++;
1806
+ ls++;
1807
+ }
1808
+ if (ind <= floor) {
1809
+ return 0;
1810
+ }
1811
+ }
1475
1812
  return 1;
1476
1813
  }
1477
1814
 
@@ -1490,6 +1827,7 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1490
1827
  * builder's close, then commit or roll back. ---- */
1491
1828
  if (e->ev_scopes == 0 && e->sink != NULL && e->sink->on_flow_build != NULL) {
1492
1829
  size_t bclose = 0;
1830
+ uint32_t serial_at_open = e->anchor_serial;
1493
1831
  int rc = e->sink->on_flow_build(e->sink->ctx, p, open_pos, len, e->line, e->line_start,
1494
1832
  anchor, tag, anchor_id, e->max_depth, &bclose);
1495
1833
  if (rc < 0) {
@@ -1509,6 +1847,14 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1509
1847
  if (e->sink->on_flow_commit(e->sink->ctx) != 1) {
1510
1848
  return -2;
1511
1849
  }
1850
+ /* the commit places the whole subtree at once: every anchor
1851
+ * defined during the flow's parse belongs to it (the builder
1852
+ * placed their nodes; the engine never sees the ids
1853
+ * individually) — mark the range so a later alias may parse */
1854
+ e_anchor_mark(e, anchor_id);
1855
+ for (uint32_t aid = serial_at_open + 1; aid <= e->anchor_serial; aid++) {
1856
+ e_anchor_mark(e, aid);
1857
+ }
1512
1858
  size_t from = open_pos;
1513
1859
  uint32_t ln = e->line;
1514
1860
  size_t ls = e->line_start;
@@ -1530,6 +1876,12 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1530
1876
  yep_json_walk w;
1531
1877
  yep_json_tok t;
1532
1878
  yep_json_walk_init(&w, p, len, open_pos, e->max_depth);
1879
+ w.strict = 1; /* the YAML fast path is byte-for-byte JSON only:
1880
+ * a lenient walk accepted number/literal keys
1881
+ * ("{9:1}" -> {9->1}) that the general kernel
1882
+ * reads as the plain scalar "9:1" — whole and
1883
+ * chunked parses took different paths and
1884
+ * diverged (the nightly's run 38513849602) */
1533
1885
  for (;;) {
1534
1886
  yep_jw_status st = yep_json_walk_next(&w, &t);
1535
1887
  if (st == YEP_JW_REJECT) {
@@ -1548,7 +1900,13 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1548
1900
  * skips the flow_enforce rescan AND the per-event line bookkeeping
1549
1901
  * below; both are no-ops without a newline, and pass 1 already
1550
1902
  * walked these bytes (TODO.restructure/45, Phase B slice). */
1551
- int single_line = (memchr(p + open_pos, '\n', close - open_pos) == NULL);
1903
+ /* a span carrying ANY \r is multi-line for marking purposes: the
1904
+ * lone-\r walk below counts real breaks, and a single-line span
1905
+ * skips that walk entirely — a '{\r\r\r\r}' span kept its marks
1906
+ * on line 1 while the general kernel (and libyaml) counted four
1907
+ * breaks (the nightly's run 38585830922) */
1908
+ int single_line = (memchr(p + open_pos, '\n', close - open_pos) == NULL) &&
1909
+ (memchr(p + open_pos, '\r', close - open_pos) == NULL);
1552
1910
 
1553
1911
  if (e_flow_key_after(p, len, close)) {
1554
1912
  return 0;
@@ -1569,9 +1927,15 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1569
1927
  ev.anchor_id = anchor_id;
1570
1928
  ev.tag = tag;
1571
1929
  ev.line = cur_line;
1572
- ev.col = (uint32_t)(open_pos + 1 - cur_ls) + 1;
1930
+ /* the general kernel's opener marks, mirrored exactly: col =
1931
+ * the indicator's 1-based column, end one past it. The old +1
1932
+ * made every fused open sit one column right of the same
1933
+ * span's general-kernel parse — a <24-byte span cut at a
1934
+ * chunk boundary routes general while the whole parse routes
1935
+ * fused (the nightly's run 38173499680) */
1936
+ ev.col = (uint32_t)(open_pos - cur_ls) + 1;
1573
1937
  ev.end_line = cur_line; /* flow opens end after the indicator */
1574
- ev.end_col = (uint32_t)(open_pos + 2 - cur_ls) + 1;
1938
+ ev.end_col = (uint32_t)(open_pos - cur_ls) + 2;
1575
1939
  if (emit_now(e, &ev) != 0) {
1576
1940
  return -2;
1577
1941
  }
@@ -1618,9 +1982,16 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1618
1982
  if (!single_line)
1619
1983
  jx_advance_line(e, &cur_scan, i, &cur_line, &cur_ls);
1620
1984
  ev.line = cur_line;
1621
- ev.col = (uint32_t)(i + 1 - cur_ls) + 1;
1985
+ /* the opener's own column, like site 1 and the general
1986
+ * kernel: the fused walk's nested open rode the OLD +1
1987
+ * (one past the indicator) while the outer opener was
1988
+ * already fixed -- whole parses >=24 bytes route fused,
1989
+ * the <24-byte chunked prefix routes general, and the
1990
+ * nested start sat one column right (the nightly's round
1991
+ * 122, "[[]": whole 1:3 vs chunk 1:2) */
1992
+ ev.col = (uint32_t)(i - cur_ls) + 1;
1622
1993
  ev.end_line = cur_line; /* flow opens end after the indicator */
1623
- ev.end_col = (uint32_t)(i + 2 - cur_ls) + 1;
1994
+ ev.end_col = (uint32_t)(i + 1 - cur_ls) + 1;
1624
1995
  if (emit_now(e, &ev) != 0) {
1625
1996
  return -2;
1626
1997
  }
@@ -1686,14 +2057,6 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1686
2057
  ev.style = YEP_STYLE_PLAIN;
1687
2058
  ev.implicit = 1;
1688
2059
  }
1689
- /* a map's first scalar of a pair is a KEY: the 1024-character
1690
- * simple-key limit applies (raw span, quotes included) */
1691
- if (stk[sd - 1] && kpending[sd - 1] == 0) {
1692
- size_t kspan = raw_end - vstart;
1693
- if (kspan > YEP_MAX_SIMPLE_KEY) {
1694
- return e_fail(e, YEP_ERR_KEY_TOO_LONG, vstart);
1695
- }
1696
- }
1697
2060
  if (raw_end == vstart) {
1698
2061
  raw_end = i; /* numbers/literals advance i, not raw_end */
1699
2062
  }
@@ -1703,6 +2066,15 @@ static int e_flow_json(yep_engine* e, yep_view anchor, yep_view tag, uint32_t an
1703
2066
  return -2;
1704
2067
  }
1705
2068
  if (stk[sd - 1] && p[i] == ':' && kpending[sd - 1] == 0) {
2069
+ /* the simple-key law rides the colon: libyaml keeps a flow
2070
+ * key possible while (colon - key start) <= 1024 in
2071
+ * characters -- the quotes count against quoted keys and a
2072
+ * multibyte code point counts once (the nightly's round 108
2073
+ * shrink carries a 2-byte char at byte delta 1025) */
2074
+ if (i - vstart > YEP_MAX_SIMPLE_KEY &&
2075
+ yep_simple_key_chars(p + vstart, i - vstart) > YEP_MAX_SIMPLE_KEY) {
2076
+ return e_fail(e, YEP_ERR_KEY_TOO_LONG, vstart);
2077
+ }
1706
2078
  kpending[sd - 1] = 2; /* key seen; the next scalar/collection is the value */
1707
2079
  }
1708
2080
  }
@@ -1727,6 +2099,8 @@ static void e_flow_frame_init(void* stv, int idx, int kind, int pending_key) {
1727
2099
  uint8_t pair_open;
1728
2100
  uint8_t pair_value;
1729
2101
  uint8_t pair_wrap;
2102
+ uint8_t q_coll; /* seq: an explicit '?' key collection awaits completion */
2103
+ uint8_t key_coll; /* map: the nested collection opened in key position */
1730
2104
  int32_t buf_from;
1731
2105
  uint32_t entry_line;
1732
2106
  uint32_t entries;
@@ -1736,9 +2110,11 @@ static void e_flow_frame_init(void* stv, int idx, int kind, int pending_key) {
1736
2110
  st[idx].pending_key = (uint8_t)pending_key;
1737
2111
  st[idx].sep = 0;
1738
2112
  st[idx].just_closed = 0;
2113
+ st[idx].key_coll = 0;
1739
2114
  st[idx].pair_open = 0;
1740
2115
  st[idx].pair_value = 0;
1741
2116
  st[idx].pair_wrap = 0;
2117
+ st[idx].q_coll = 0;
1742
2118
  st[idx].buf_from = -1;
1743
2119
  st[idx].entry_line = 0;
1744
2120
  st[idx].entries = 0;
@@ -1753,6 +2129,8 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1753
2129
  uint8_t pair_open; /* seq: a buffered single-pair awaits its MAP_END */
1754
2130
  uint8_t pair_value; /* seq: the open pair already has its value */
1755
2131
  uint8_t pair_wrap; /* map frame pushed as a seq single-pair */
2132
+ uint8_t q_coll; /* seq: an explicit '?' key collection awaits completion */
2133
+ uint8_t key_coll; /* map: a nested collection opened in key position */
1756
2134
  int32_t buf_from; /* seq: entry events buffered from this index */
1757
2135
  uint32_t entry_line; /* seq: line where the buffered entry ended */
1758
2136
  uint32_t entries; /* completed entries (for trailing-comma detection) */
@@ -1863,7 +2241,24 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1863
2241
  if (n >= e->max_depth) {
1864
2242
  return e_fail(e, YEP_ERR_DEPTH, e->pos);
1865
2243
  }
1866
- st[n - 1].pair_value = 1;
2244
+ if (q_key) {
2245
+ /* the EXPLICIT key is the collection: mark its frame so
2246
+ * the pair completes with a null value when no ':' line
2247
+ * follows ("? [a]" / "? {b: 1}") — without the marker the
2248
+ * key leaked as a plain seq entry and a map-entry key
2249
+ * vanished outright (the nightly's round 101) */
2250
+ if (st[n - 1].kind == 1) {
2251
+ st[n - 1].pending_key = 2;
2252
+ } else {
2253
+ st[n - 1].q_coll = 1;
2254
+ }
2255
+ }
2256
+ if (!q_key) {
2257
+ /* a VALUE collection completes the pending pair; a KEY
2258
+ * collection must not (its pair completes at ':' or the
2259
+ * boundary with a null value) */
2260
+ st[n - 1].pair_value = 1;
2261
+ }
1867
2262
  unsigned char oc = (unsigned char)e->p[e->pos];
1868
2263
  e->pos++;
1869
2264
  yep_view pa = ev.anchor, pt = ev.tag; /* props before the opener */
@@ -1883,9 +2278,11 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1883
2278
  st[n - 1].just_closed = 0;
1884
2279
  /* the nested collection is an entry of the parent frame; when
1885
2280
  * it appears in a map's value position it completes the pair */
2281
+ int coll_key = (st[n - 1].kind == 1 && st[n - 1].pending_key == 0 && !q_key);
1886
2282
  if (st[n - 1].kind == 1 && st[n - 1].pending_key == 1) {
1887
2283
  st[n - 1].pending_key = 0;
1888
2284
  }
2285
+ st[n - 1].key_coll = (uint8_t)coll_key;
1889
2286
  st[n - 1].sep = 0;
1890
2287
  st[n - 1].entries++;
1891
2288
  e_flow_frame_init(st, n, (oc == '[') ? 0 : 1, 0);
@@ -1906,6 +2303,8 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1906
2303
  * may precede it: "unquoted : value", "\"foo\"\n: bar").
1907
2304
  * After a QUOTED node the colon separates JSON-style. */
1908
2305
  size_t save_pos = e->pos;
2306
+ uint32_t save_line = e->line;
2307
+ size_t save_ls = e->line_start;
1909
2308
  if (e_flow_ws(e) != 0) {
1910
2309
  return -1;
1911
2310
  } /* crosses newlines; restored below when no colon */
@@ -1926,13 +2325,19 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1926
2325
  e->p[e->pos + 1] == '\n' || e->p[e->pos + 1] == '\r' ||
1927
2326
  yep_ct_is((unsigned char)e->p[e->pos + 1], YEP_CT_FLOW_IND)));
1928
2327
  if (!colon) {
1929
- e->pos = save_pos; /* not a key; the caller re-reads */
2328
+ /* not a key; the caller re-reads — the line state goes
2329
+ * back with the cursor (the ws lookahead crossed the
2330
+ * newline; a stale e->line marked everything after the
2331
+ * flow node one line low) */
2332
+ e->pos = save_pos;
2333
+ e->line = save_line;
2334
+ e->line_start = save_ls;
1930
2335
  }
1931
2336
  if (colon && st[n - 1].kind == 1 && st[n - 1].pending_key == 0) {
1932
- if (!q_key && quoted && ev.value.len + 2 > YEP_MAX_SIMPLE_KEY) {
1933
- return e_fail(e, YEP_ERR_KEY_TOO_LONG, e->pos);
1934
- }
1935
- if (!q_key && !quoted && ev.value.len > YEP_MAX_SIMPLE_KEY) {
2337
+ if (!q_key && ((quoted && yep_simple_key_chars(ev.value.p, ev.value.len) + 2 >
2338
+ YEP_MAX_SIMPLE_KEY) ||
2339
+ (!quoted && yep_simple_key_chars(ev.value.p, ev.value.len) >
2340
+ YEP_MAX_SIMPLE_KEY))) {
1936
2341
  return e_fail(e, YEP_ERR_KEY_TOO_LONG, e->pos);
1937
2342
  }
1938
2343
  e->pos++;
@@ -1944,10 +2349,18 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
1944
2349
  continue;
1945
2350
  }
1946
2351
  if (colon && st[n - 1].kind == 0) {
1947
- if (!q_key && quoted && ev.value.len + 2 > YEP_MAX_SIMPLE_KEY) {
1948
- return e_fail(e, YEP_ERR_KEY_TOO_LONG, e->pos);
2352
+ if (st[n - 1].pair_open && st[n - 1].pair_value) {
2353
+ /* "[s: 2:]" — the open pair already has its value;
2354
+ * a second single-pair would nest inside the value
2355
+ * slot and unbalance the event stream (the nightly's
2356
+ * run 38168927841). libyaml rejects the ':' mid-
2357
+ * scalar; so does the suite's flow grammar. */
2358
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
1949
2359
  }
1950
- if (!q_key && !quoted && ev.value.len > YEP_MAX_SIMPLE_KEY) {
2360
+ if (!q_key && ((quoted && yep_simple_key_chars(ev.value.p, ev.value.len) + 2 >
2361
+ YEP_MAX_SIMPLE_KEY) ||
2362
+ (!quoted && yep_simple_key_chars(ev.value.p, ev.value.len) >
2363
+ YEP_MAX_SIMPLE_KEY))) {
1951
2364
  return e_fail(e, YEP_ERR_KEY_TOO_LONG, e->pos);
1952
2365
  }
1953
2366
  if (st[n - 1].buf_from >= 0 && !quoted && !q_key &&
@@ -2013,11 +2426,11 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2013
2426
  return brc;
2014
2427
  }
2015
2428
  }
2016
- yep_event nv;
2017
- e_event_init(&nv, YEP_EV_SCALAR);
2018
- nv.implicit = 1;
2019
- nv.style = YEP_STYLE_PLAIN;
2020
- if (emit_now(e, &nv) != 0) {
2429
+ /* the parsed key rides the slot the empty stand-in
2430
+ * occupied — dropping it lost the key's content AND
2431
+ * its anchor (an alias to it then failed the
2432
+ * placed-anchor check: the nightly's round 101) */
2433
+ if (emit_now(e, &ev) != 0) {
2021
2434
  return -2;
2022
2435
  }
2023
2436
  continue;
@@ -2054,6 +2467,22 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2054
2467
  unsigned char c = (unsigned char)e->p[e->pos];
2055
2468
  if (c == ',') {
2056
2469
  e->pos++;
2470
+ if (st[n - 1].q_coll && st[n - 1].buf_from >= 0) {
2471
+ int32_t bf0 = st[n - 1].buf_from;
2472
+ st[n - 1].buf_from = -1;
2473
+ e->ev_scopes--;
2474
+ if (e_buf_wrap(e, (uint32_t)bf0) != 0) {
2475
+ return -2;
2476
+ }
2477
+ if (e->ev_scopes == 0 && e->ev_buf_n > 0) {
2478
+ int brc = e_buf_send(e, 0);
2479
+ if (brc != 0) {
2480
+ return brc;
2481
+ }
2482
+ }
2483
+ st[n - 1].q_coll = 0;
2484
+ st[n - 1].pair_open = 1;
2485
+ }
2057
2486
  if (st[n - 1].pair_open) {
2058
2487
  if (!st[n - 1].pair_value) {
2059
2488
  yep_event nv;
@@ -2099,11 +2528,39 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2099
2528
  }
2100
2529
  st[n - 1].pending_key = 0;
2101
2530
  }
2531
+ if (st[n - 1].kind == 1 && st[n - 1].key_coll && st[n - 1].pending_key == 0) {
2532
+ /* "{[a], z: 1}" — a closed collection with no ':' is a
2533
+ * bare key: libyaml emits the implicit null value */
2534
+ yep_event nv;
2535
+ e_event_init(&nv, YEP_EV_SCALAR);
2536
+ nv.implicit = 1;
2537
+ nv.style = YEP_STYLE_PLAIN;
2538
+ if (emit_now(e, &nv) != 0) {
2539
+ return -2;
2540
+ }
2541
+ st[n - 1].key_coll = 0;
2542
+ }
2102
2543
  st[n - 1].sep = 1;
2103
2544
  st[n - 1].just_closed = 0;
2104
2545
  continue;
2105
2546
  }
2106
2547
  if (c == ']' || c == '}') {
2548
+ if (st[n - 1].q_coll && st[n - 1].buf_from >= 0) {
2549
+ int32_t bf0 = st[n - 1].buf_from;
2550
+ st[n - 1].buf_from = -1;
2551
+ e->ev_scopes--;
2552
+ if (e_buf_wrap(e, (uint32_t)bf0) != 0) {
2553
+ return -2;
2554
+ }
2555
+ if (e->ev_scopes == 0 && e->ev_buf_n > 0) {
2556
+ int brc = e_buf_send(e, 0);
2557
+ if (brc != 0) {
2558
+ return brc;
2559
+ }
2560
+ }
2561
+ st[n - 1].q_coll = 0;
2562
+ st[n - 1].pair_open = 1;
2563
+ }
2107
2564
  while (st[n - 1].pair_wrap) {
2108
2565
  /* a single-pair wrapper inside a sequence closes on any
2109
2566
  * bracket ("[ : v ]", CFD4) */
@@ -2146,6 +2603,19 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2146
2603
  }
2147
2604
  st[n - 1].pending_key = 0;
2148
2605
  }
2606
+ if (st[n - 1].kind == 1 && st[n - 1].key_coll && st[n - 1].pending_key == 0) {
2607
+ /* "{{}}" / "{[a]}" — the closed collection is a bare
2608
+ * key: libyaml emits the implicit null value before
2609
+ * the frame's own MAP_END */
2610
+ yep_event nv;
2611
+ e_event_init(&nv, YEP_EV_SCALAR);
2612
+ nv.implicit = 1;
2613
+ nv.style = YEP_STYLE_PLAIN;
2614
+ if (emit_now(e, &nv) != 0) {
2615
+ return -2;
2616
+ }
2617
+ st[n - 1].key_coll = 0;
2618
+ }
2149
2619
  if (st[n - 1].pair_open) {
2150
2620
  if (!st[n - 1].pair_value) {
2151
2621
  yep_event nv;
@@ -2217,6 +2687,7 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2217
2687
  if (st[n - 1].kind == 1 && st[n - 1].just_closed) {
2218
2688
  /* '[a, b]: v' inside a map — the closed collection is the key */
2219
2689
  st[n - 1].just_closed = 0;
2690
+ st[n - 1].key_coll = 0;
2220
2691
  st[n - 1].pending_key = 1;
2221
2692
  continue;
2222
2693
  }
@@ -2234,11 +2705,18 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2234
2705
  continue;
2235
2706
  }
2236
2707
  if (st[n - 1].kind == 0 && st[n - 1].buf_from >= 0) {
2708
+ if (st[n - 1].pair_open && st[n - 1].pair_value) {
2709
+ /* "[s: 2\n:]" — the loop-top twin of the inline
2710
+ * colon guard: a buffered value cannot wrap as
2711
+ * another pair once the open pair is complete */
2712
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
2713
+ }
2237
2714
  if (st[n - 1].entry_line != e->line) {
2238
2715
  /* the key ended on an earlier line */
2239
2716
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
2240
2717
  }
2241
2718
  /* '[ [a, b]: v' — the buffered collection is the pair key */
2719
+ st[n - 1].q_coll = 0;
2242
2720
  int32_t bf0 = st[n - 1].buf_from;
2243
2721
  int was_empty = ((uint32_t)bf0 == e->ev_buf_n);
2244
2722
  st[n - 1].buf_from = -1;
@@ -2264,6 +2742,12 @@ static int e_flow(yep_engine* e, yep_view anchor, yep_view tag, uint32_t anchor_
2264
2742
  continue; /* value next */
2265
2743
  }
2266
2744
  if (st[n - 1].kind == 0 && n >= 1) {
2745
+ if (st[n - 1].pair_open) {
2746
+ /* "[s: : v]" — ':' in the open pair's value slot
2747
+ * is not node content (libyaml: "did not find
2748
+ * expected node content") */
2749
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
2750
+ }
2267
2751
  /* '[ : v' — single-pair mapping with an empty key */
2268
2752
  if (n >= e->max_depth) {
2269
2753
  return e_fail(e, YEP_ERR_DEPTH, e->pos);
@@ -2568,6 +3052,12 @@ static int e_classified(yep_engine* e, uint16_t floor_col) {
2568
3052
  if (prepared) {
2569
3053
  if (e->sink != NULL && e->sink->on_block_pair != NULL) {
2570
3054
  yep_view key = {e->p + sh->key_start, sh->key_end - sh->key_start};
3055
+ /* the fused placement places the value node: its anchor
3056
+ * rides v — mark it so a later alias may parse (the
3057
+ * placement sites are the alias check's ground truth; the
3058
+ * classified KEY is always plain — anchored keys ride the
3059
+ * general chain) */
3060
+ e_anchor_mark(e, v.anchor_id);
2571
3061
  int built =
2572
3062
  e->sink->on_block_pair(e->sink->ctx, &key, &v, pair_line, key_col, pair_val_col);
2573
3063
  if (built < 0) {
@@ -2691,6 +3181,33 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2691
3181
 
2692
3182
  if (e_at_eol(e)) {
2693
3183
  /* properties with the value on following lines */
3184
+ if (e->depth > 0 && e->frames[e->depth - 1].kind == YEP_FRAME_MAP &&
3185
+ e->frames[e->depth - 1].col == e_col(e, node_at) &&
3186
+ (!yep_view_is_empty(anchor) || !yep_view_is_empty(tag))) {
3187
+ /* a props-only line at an ESTABLISHED pair position: the
3188
+ * scanner never finds the simple key's ':' (libyaml:
3189
+ * "could not find expected ':' while scanning a simple
3190
+ * key"; Psych rejects "k: 1\n!!t: &b\n n: m" and the
3191
+ * anchor-only twins). The old path consumed the line as
3192
+ * node props — the pair's key event vanished, the key's
3193
+ * tag leaked onto the value node, and a long-tag value
3194
+ * then lost its anchor at emit (the nightly's run
3195
+ * 38275208009). The FIRST pair of a fresh map ("a:\n
3196
+ * !!t: &c\n n: m") has no frame at the column yet and
3197
+ * stays legal. */
3198
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
3199
+ }
3200
+ if (!yep_view_is_empty(pend_a) && !yep_view_is_empty(anchor)) {
3201
+ /* a second props line carrying its own ANCHOR displaces the
3202
+ * pend anchor (same-line-wins) — the displaced anchor stays
3203
+ * in the nametab, so a later alias emits a dangling
3204
+ * reference (the nightly's run 38271222289, "&a\n&a1\nF:
3205
+ * *a" and its '? &a' explicit-key twin). libyaml rejects
3206
+ * every nesting variant ("did not find expected <document
3207
+ * start>" / "did not find expected key"); a TAG-only
3208
+ * continuation ("&a\n!!str v") stays legal. */
3209
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
3210
+ }
2694
3211
  e->pend_anchor = node_a;
2695
3212
  e->pend_anchor_id = anchor_ordinal;
2696
3213
  e->pend_tag = node_t;
@@ -2730,8 +3247,24 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2730
3247
 
2731
3248
  if (c == '?' && (e->pos + 1 >= e->len || e->p[e->pos + 1] == ' ' || e->p[e->pos + 1] == '\t' ||
2732
3249
  e->p[e->pos + 1] == '\n' || e->p[e->pos + 1] == '\r')) {
2733
- if (ctx == YEP_CTX_AFTER_COLON) {
2734
- return e_fail(e, YEP_ERR_UNEXPECTED, e->pos); /* "key: ? a" */
3250
+ if (ctx == YEP_CTX_AFTER_COLON || e->cv_line == e->line) {
3251
+ /* "key: ? a" and ": ?" — a key indicator where a value
3252
+ * belongs (libyaml rejects); accepting it produced an
3253
+ * unpairable event stream whose orphaned anchored node
3254
+ * the emit dropped (the nightly's round 102). The
3255
+ * main-dispatch ':' line rides AFTER_DASH, so the
3256
+ * colon's own line marks it. */
3257
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
3258
+ }
3259
+ if (!yep_view_is_empty(anchor) || !yep_view_is_empty(tag)) {
3260
+ /* node properties followed by a second key indicator on
3261
+ * the SAME line: libyaml "mapping keys are not allowed in
3262
+ * this context" (psych rejects "? &a ? b" and "k: &a ? b";
3263
+ * the cross-line twin "? &a\n ? b" composes and stays
3264
+ * legal). The old path opened the nested key map with the
3265
+ * props dropped — a later alias then emitted a dangling
3266
+ * reference (the nightly's run 38355778230). */
3267
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
2735
3268
  }
2736
3269
  if (e->pos > 0 && e->p[e->pos - 1] == '\t') {
2737
3270
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos); /* "?\t-" (Y79Y) */
@@ -2741,7 +3274,33 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2741
3274
  if (rc != 0) {
2742
3275
  return rc;
2743
3276
  }
2744
- e->q_map_depth = e->depth;
3277
+ if (e->qmd_n >= 16) {
3278
+ return e_fail(e, YEP_ERR_DEPTH, e->pos); /* the depth-guard law */
3279
+ }
3280
+ /* a pending from an EARLIER line at this level completes with a
3281
+ * null value before the new pair opens — without the flush the
3282
+ * two pairs share one wrapper and the merge model leaves the
3283
+ * later value an unpairable orphan (the nightly's round 102) */
3284
+ /* a key indicator at the pending pair's own depth from a LATER
3285
+ * line completes that pair with a null value (libyaml's
3286
+ * same-indent KEY rule). The pending may still be mid-content
3287
+ * (this '?' is being parsed as its content): arm the flag the
3288
+ * flush keys on — the outer branch re-arms only if its slot
3289
+ * survived. Without the flush the outer pair's value never
3290
+ * emits and the DOM drops the unpaired node (the nightly's
3291
+ * round 102). */
3292
+ if (e->qmd_n > 0 && e->qmd_stack[e->qmd_n - 1] == e->depth &&
3293
+ e->qmd_line[e->qmd_n - 1] != e->line) {
3294
+ if (!e->q_value_pending) {
3295
+ e->q_value_pending = 1;
3296
+ }
3297
+ int frc = e_flush_q_value(e);
3298
+ if (frc != 0) {
3299
+ return frc;
3300
+ }
3301
+ }
3302
+ e->qmd_line[e->qmd_n] = e->line;
3303
+ e->qmd_stack[e->qmd_n++] = e->depth;
2745
3304
  e->pos++;
2746
3305
  e_skip_inline_space(e);
2747
3306
  if (e->pos < e->len && e->p[e->pos] == ':' &&
@@ -2762,6 +3321,9 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2762
3321
  return -2;
2763
3322
  }
2764
3323
  e->pos++;
3324
+ if (e->qmd_n > 0) {
3325
+ e->qmd_n--; /* the ':' line supplied the innermost value */
3326
+ }
2765
3327
  return e_parse_value(e, YEP_CTX_AFTER_COLON, col);
2766
3328
  }
2767
3329
  /* '? : x' — the ':' starts a nested pair: the KEY is that map */
@@ -2770,11 +3332,17 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2770
3332
  * its content may ALIGN with the '?' column */
2771
3333
  e->q_key_pending = 1;
2772
3334
  {
3335
+ int my_qmd = e->qmd_n; /* this pair's stack slot is my_qmd - 1 */
2773
3336
  int qrc = e_parse_value(e, YEP_CTX_AFTER_Q, col);
2774
3337
  if (qrc != 0) {
2775
3338
  return qrc;
2776
3339
  }
2777
- e->q_value_pending = 1; /* until a ':' line supplies the value */
3340
+ /* a later-line '?' completed this pair inside the content
3341
+ * parse (the flush popped our slot): the flag must not
3342
+ * re-arm, or the stale pending emits a stray null */
3343
+ if (e->qmd_n >= my_qmd) {
3344
+ e->q_value_pending = 1; /* until a ':' line supplies the value */
3345
+ }
2778
3346
  return 0;
2779
3347
  }
2780
3348
  }
@@ -2804,12 +3372,20 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2804
3372
  }
2805
3373
  size_t save = e->pos;
2806
3374
  uint32_t line0 = e->line;
3375
+ size_t ls0 = e->line_start;
2807
3376
  if (e_skip_flow(e) != 0) {
2808
3377
  return -1;
2809
3378
  }
2810
3379
  e_skip_inline_space(e);
2811
3380
  int is_key = e_colon_at(e, e->pos) && e->line == line0;
3381
+ /* the pre-scan tracks lines while it crosses them; the re-scan
3382
+ * below re-derives every mark from the open position, so the
3383
+ * line state must go back with the cursor (the nightly's run
3384
+ * 38280372447: a multi-line flow's open event marked at the
3385
+ * pre-scan's far line, diverging from the JSON fast path) */
2812
3386
  e->pos = save;
3387
+ e->line = line0;
3388
+ e->line_start = ls0;
2813
3389
  if (is_key && ctx == YEP_CTX_AFTER_COLON) {
2814
3390
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
2815
3391
  }
@@ -2822,12 +3398,15 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2822
3398
  if (rc != 0) {
2823
3399
  return rc;
2824
3400
  }
2825
- rc = e_flow(e, node_a, node_t, node_aid);
3401
+ /* the pend props went to the mapping; the key collection
3402
+ * keeps only same-line props (Psych: "!t\n[a]: v" tags
3403
+ * the map, "!t [a]: v" tags the key) */
3404
+ rc = e_flow(e, anchor, tag, anchor_ordinal);
2826
3405
  if (rc != 0) {
2827
3406
  return rc;
2828
3407
  }
2829
3408
  e->pos++; /* ':' */
2830
- return e_parse_value(e, YEP_CTX_AFTER_COLON, key_col);
3409
+ return e_pair_value_done(e, ctx, key_col);
2831
3410
  }
2832
3411
  {
2833
3412
  e->flow_floor = floor_col;
@@ -2863,11 +3442,19 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2863
3442
  if (rc != 0) {
2864
3443
  return rc;
2865
3444
  }
3445
+ /* the pend props went to the mapping; the key keeps only
3446
+ * same-line props — ev carries the merged views and would
3447
+ * leak the map's tag onto the key (the nightly's
3448
+ * byte-instability: "!<!>\n\"\":\n" re-parsed with a
3449
+ * tagged first key and never re-stabilized) */
3450
+ ev.anchor = anchor;
3451
+ ev.anchor_id = anchor_ordinal;
3452
+ ev.tag = tag;
2866
3453
  if (emit_now(e, &ev) != 0) {
2867
3454
  return -2;
2868
3455
  }
2869
3456
  e->pos++; /* ':' */
2870
- return e_parse_value(e, YEP_CTX_AFTER_COLON, key_col);
3457
+ return e_pair_value_done(e, ctx, key_col);
2871
3458
  }
2872
3459
  if (!e_at_eol(e)) {
2873
3460
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
@@ -2898,7 +3485,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2898
3485
  return -2;
2899
3486
  }
2900
3487
  e->pos++;
2901
- return e_parse_value(e, YEP_CTX_AFTER_COLON, key_col);
3488
+ return e_pair_value_done(e, ctx, key_col);
2902
3489
  }
2903
3490
  if (!e_at_eol(e)) {
2904
3491
  return e_fail(e, YEP_ERR_UNEXPECTED, e->pos);
@@ -2976,7 +3563,7 @@ static int e_node(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
2976
3563
  e->pos++;
2977
3564
  }
2978
3565
  e->pos++; /* past ':' */
2979
- return e_parse_value(e, YEP_CTX_AFTER_COLON, key_col);
3566
+ return e_pair_value_done(e, ctx, key_col);
2980
3567
  }
2981
3568
 
2982
3569
  ev.value.p = e->p + s.start;
@@ -3072,25 +3659,39 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
3072
3659
  goto empty_value;
3073
3660
  }
3074
3661
  if (li.indent == floor_col && e->depth > 0 &&
3075
- (ctx == YEP_CTX_AFTER_COLON || ctx == YEP_CTX_VALUE_LINE) &&
3662
+ (ctx == YEP_CTX_AFTER_COLON || ctx == YEP_CTX_VALUE_LINE || ctx == YEP_CTX_AFTER_Q) &&
3076
3663
  (li.first == '&' || li.first == '!')) {
3077
3664
  /* Only PROPS-ONLY lines are new nodes, not values
3078
- * ("key: &x\n!!map\na: b"); "&anchor c: 3" is a keyed pair */
3665
+ * ("key: &x\n!!map\na: b"); "&anchor c: 3" is a keyed pair.
3666
+ * UNDER AN EXPLICIT '?' KEY a props-only line is worse: the
3667
+ * key would span lines waiting for a ':' that its own
3668
+ * props absorb — libyaml rejects ("could not find expected
3669
+ * ':' while scanning a simple key"), and absorbing the
3670
+ * props DROPPED the key's anchor so the emission wrote an
3671
+ * alias to an anchor that no longer existed ("? &a\n&F\n:
3672
+ * *a", the nightly's run 38207965687) */
3079
3673
  size_t t = li.offset + li.indent;
3080
3674
  int only_props = 1;
3675
+ char prev = ' '; /* '#' is a comment only after a blank —
3676
+ * mid-token it is content ("&a ?#:" is a
3677
+ * keyed pair whose plain carries a '#',
3678
+ * the nightly's run 38204470906) */
3081
3679
  while (t < li.end) {
3082
3680
  char pc = e->p[t];
3083
3681
  if (pc == ' ' || pc == '\t') {
3682
+ prev = pc;
3084
3683
  t++;
3085
3684
  continue;
3086
3685
  }
3686
+ int pc_comment = (pc == '#' && (prev == ' ' || prev == '\t'));
3087
3687
  if (pc == '&' || pc == '!' || pc == '<' || pc == ',' || pc == '>' ||
3088
- (pc != '\n' && pc != '\r' && pc != '#' &&
3688
+ (pc != '\n' && pc != '\r' && !pc_comment &&
3089
3689
  !yep_ct_is((unsigned char)pc, YEP_CT_FLOW_IND) && pc != ':' && pc != ' ')) {
3690
+ prev = pc;
3090
3691
  t++;
3091
3692
  continue;
3092
3693
  }
3093
- only_props = (pc == '\n' || pc == '\r' || pc == '#');
3694
+ only_props = (pc == '\n' || pc == '\r' || pc_comment);
3094
3695
  break;
3095
3696
  }
3096
3697
  if (only_props) {
@@ -3102,6 +3703,53 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
3102
3703
  * at any indent is the value ("&a\n- x"). Explicit-key content
3103
3704
  * may align with the '?' column. Flow nodes and property runs
3104
3705
  * continue the value even at the parent column ("k:\n!!seq\n[a]"). */
3706
+ if (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first == '-' &&
3707
+ li.offset + 1 < li.end && e->p[li.offset + 1] != ' ' && e->p[li.offset + 1] != '\t' &&
3708
+ e->p[li.offset + 1] != '\n' && e->p[li.offset + 1] != '\r') {
3709
+ /* "-:" and friends: a dash with a non-space follower is a
3710
+ * PLAIN key ("-" => null), never the pending key's content
3711
+ * ("?\n-:" lost the null pair, the nightly's run
3712
+ * 39045959147). A dash+space seq entry keeps the old
3713
+ * content path below (the catch-38154662070 shape). */
3714
+ goto empty_value;
3715
+ }
3716
+ if (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first != ':' &&
3717
+ li.first != '-' && yep_plain_first_ok(li.first)) {
3718
+ /* a same-column PLAIN-START pair line is a SIBLING, not
3719
+ * the key's content: the buffered "? &a" key would strand
3720
+ * inside it and an alias to the anchor emitted a dangling
3721
+ * reference ("? &a\nt: &b\n*a"; the nightly's run
3722
+ * 38261610465; libyaml: "could not find expected ':'
3723
+ * while scanning a simple key"). Dashes keep the old path
3724
+ * (a block seq is the key's content — the catch-38154662070
3725
+ * pin's shape). Stop: the null value flushes and the main
3726
+ * loop owns the pair. */
3727
+ if (li.first == '?') {
3728
+ /* a '?'-led line at the key's own column is NEVER the
3729
+ * key's content: an indicator ("? k") or a plain-key
3730
+ * pair ("?:") alike, libyaml completes the pending key
3731
+ * (null value) and the main loop re-dispatches the
3732
+ * line. The old continuation swallowed it as a folded
3733
+ * piece and the pending pair lost its null value
3734
+ * ("?\n?:" parsed as one pair, libyaml two: the
3735
+ * nightly's run 39041682907) */
3736
+ goto empty_value;
3737
+ }
3738
+ size_t sc = li.offset + li.indent;
3739
+ int sibling = 0;
3740
+ while (sc < li.end) {
3741
+ if (e->p[sc] == ':' &&
3742
+ (sc + 1 >= (size_t)li.end || e->p[sc + 1] == ' ' || e->p[sc + 1] == '\t' ||
3743
+ e->p[sc + 1] == '\n' || e->p[sc + 1] == '\r')) {
3744
+ sibling = 1;
3745
+ break;
3746
+ }
3747
+ sc++;
3748
+ }
3749
+ if (sibling) {
3750
+ goto empty_value;
3751
+ }
3752
+ }
3105
3753
  if (li.indent > floor_col || e->depth == 0 ||
3106
3754
  (ctx == YEP_CTX_AFTER_Q && li.indent == floor_col && li.first != ':') ||
3107
3755
  ((li.first == '[' || li.first == '{') && li.indent == floor_col) ||
@@ -3112,6 +3760,17 @@ static int e_parse_value(yep_engine* e, yep_ctx ctx, uint16_t floor_col) {
3112
3760
  * fixture shape) */
3113
3761
  (vctx == YEP_CTX_VALUE_LINE && (li.first == '|' || li.first == '>') &&
3114
3762
  li.indent == floor_col)) {
3763
+ /* an empty-key ':' line cannot be a pending value's
3764
+ * content — libyaml errors ("did not find expected key";
3765
+ * the nightly's "- &a\n :" byte-instability crasher).
3766
+ * The same-line empty-key law lives in the shape arms. */
3767
+ if (li.first == ':') {
3768
+ size_t at2 = e->pos + li.indent + 1;
3769
+ if (at2 >= e->len || e->p[at2] == ' ' || e->p[at2] == '\t' || e->p[at2] == '\n' ||
3770
+ e->p[at2] == '\r') {
3771
+ return e_fail(e, YEP_ERR_UNEXPECTED, e->pos + li.indent);
3772
+ }
3773
+ }
3115
3774
  e->pos += li.indent;
3116
3775
  if (++e->rec >= e->max_depth) {
3117
3776
  e->rec--;
@@ -3155,6 +3814,12 @@ empty_value: {
3155
3814
  { /* libyaml: the implicit null sits just past the key's colon
3156
3815
  * ("k: " -> col 2, not the line end), zero-span */
3157
3816
  size_t at = e->pos;
3817
+ if (at > e->len) {
3818
+ at = e->len; /* the cursor may stand one past EOF after a
3819
+ * flow close scan (the nightly's run
3820
+ * 38147359781, '[&"[?:\n?(\nu\n?"]:...') —
3821
+ * zero-copy means the mark reads in-bounds */
3822
+ }
3158
3823
  while (at > 0 && (e->p[at - 1] == ' ' || e->p[at - 1] == '\t' || e->p[at - 1] == '\n' ||
3159
3824
  e->p[at - 1] == '\r')) {
3160
3825
  at--; /* the cursor may sit lines past the deferred key */
@@ -3236,9 +3901,13 @@ yep_engine* yep_engine_create(const yep_allocator* sys) {
3236
3901
 
3237
3902
  /* YAML 1.2 line breaks beyond \n/\r: NEL, LS, PS. When present, the
3238
3903
  * input is copied with each normalized to a single '\n' (libyaml reader
3239
- * behavior); inputs without them keep the zero-copy path. */
3240
- static const char* e_normalize_breaks(yep_engine* e, const char* p, size_t len) {
3241
- char* out = yep_engine_normalize_breaks(e->sys, p, len, NULL);
3904
+ * behavior); inputs without them keep the zero-copy path. The copied
3905
+ * length is not strlen()'s: the INPUT may carry NUL bytes of its own,
3906
+ * and a strlen() of the copy truncates the run at the first one (the
3907
+ * nightly's run 38524928004: "\0\n---\r<U+2028>" whole-parsed as an
3908
+ * empty stream while the chunked cut skipped the copy). */
3909
+ static const char* e_normalize_breaks(yep_engine* e, const char* p, size_t len, size_t* out_len) {
3910
+ char* out = yep_engine_normalize_breaks(e->sys, p, len, out_len);
3242
3911
  if (out == NULL) {
3243
3912
  return p;
3244
3913
  }
@@ -3322,17 +3991,71 @@ typedef struct yep_stepstate {
3322
3991
  size_t scanned; /* bytes folded into quote/flow/comment state */
3323
3992
  int in_flow; /* open [ { depth */
3324
3993
  int quote; /* 0 none, else the opening quote char */
3994
+ int in_verbatim; /* inside a "!<...>" verbatim tag: cuts would
3995
+ * split the tag across the chunk boundary */
3325
3996
  int in_comment; /* inside a # comment (until the line break) */
3326
3997
  int saw_directive; /* a % line awaits its --- document */
3327
3998
  int started; /* STREAM_START emitted */
3328
3999
  uint32_t line_base; /* whole-stream lines consumed by earlier runs */
4000
+ int bs_open; /* a block scalar's content lines are pending: a
4001
+ * '>' / '|' header was seen and its terminator
4002
+ * has not */
4003
+ uint32_t bs_indent; /* the header line's indent: the content floor */
4004
+ int tok_start; /* the next word may begin here: quotes are
4005
+ * tokens only at a word start (the "A!'- %YAML"
4006
+ * plain's apostrophe is content, not a quote) */
4007
+ int word_led; /* the current word began with an indicator */
4008
+ int at_break; /* the previous position was a line break
4009
+ * (multibyte breaks leave a non-break byte as
4010
+ * p[i-1], so the '#' prev-byte test misses them) */
3329
4011
  } yep_stepstate;
3330
4012
 
3331
4013
  static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep_sink* sink,
3332
4014
  int emit_start, int emit_end);
3333
4015
 
4016
+ void yep_engine_front_done(yep_engine* e) {
4017
+ e->front_done = 1;
4018
+ }
4019
+
3334
4020
  int yep_engine_run(yep_engine* e, const char* buf, size_t len, const yep_sink* sink) {
3335
- return engine_run_impl(e, buf, len, sink, 1, 1);
4021
+ /* the encoding front-end every raw byte-taking entry owns (push,
4022
+ * pull, schema, iterparse's first chunk, the libyaml differential):
4023
+ * BOM sniff + UTF-16/32 transcode. ANY head BOM is a mark -- a lone
4024
+ * one is an empty stream, UTF-8 included (libyaml's reader; the
4025
+ * nightly's round 122 parsed a lone EF BB BF as a U+FEFF scalar
4026
+ * where libyaml ended the stream). The front-end runs ONCE per
4027
+ * engine: parse.c/values.c pre-strip and mark it, and later
4028
+ * chunked runs never re-sniff (a mid-stream U+FEFF is content --
4029
+ * iterparse's chunk heads would otherwise be eaten). */
4030
+ const char* data = buf;
4031
+ size_t data_len = len;
4032
+ if (!e->front_done) {
4033
+ e->front_done = 1;
4034
+ yep_encoding enc = YEP_ENC_UNKNOWN;
4035
+ size_t bom = yep_bom_sniff((const unsigned char*)buf, len, &enc);
4036
+ data = buf + bom;
4037
+ data_len = len - bom;
4038
+ if (enc == YEP_ENC_UTF16LE || enc == YEP_ENC_UTF16BE || enc == YEP_ENC_UTF32LE ||
4039
+ enc == YEP_ENC_UTF32BE) {
4040
+ size_t tlen = 0;
4041
+ size_t terr = 0;
4042
+ int rc = yep_transcode_to_utf8(e->sys, enc, (const unsigned char*)data, data_len,
4043
+ &e->enc_buf, &tlen, &terr);
4044
+ if (rc != 0) {
4045
+ yep_error_set(&e->err, YEP_ERR_ENCODING, 1, 1, terr, "ill-formed input encoding");
4046
+ return -1;
4047
+ }
4048
+ size_t pverr = 0;
4049
+ if (!yep_printable_validate(e->enc_buf, tlen, &pverr)) {
4050
+ yep_error_set(&e->err, YEP_ERR_ENCODING, 1, 1, pverr,
4051
+ "non-printable character at byte %zu", pverr);
4052
+ return -1;
4053
+ }
4054
+ data = (const char*)e->enc_buf;
4055
+ data_len = tlen;
4056
+ }
4057
+ }
4058
+ return engine_run_impl(e, data, data_len, sink, 1, 1);
3336
4059
  }
3337
4060
 
3338
4061
  void yep_engine_destroy(yep_engine* e) {
@@ -3346,7 +4069,9 @@ void yep_engine_destroy(yep_engine* e) {
3346
4069
  e->step = NULL;
3347
4070
  }
3348
4071
  yep_nametab_free(&e->anchors);
4072
+ yep_free(e->sys, e->anchor_placed);
3349
4073
  yep_free(e->sys, e->norm_buf);
4074
+ yep_free(e->sys, e->enc_buf);
3350
4075
  yep_pool_destroy(e->pool);
3351
4076
  yep_free(e->sys, e);
3352
4077
  }
@@ -3370,12 +4095,6 @@ void yep_engine_destroy(yep_engine* e) {
3370
4095
  /* A quote opens a scalar only where a value can begin; anywhere else
3371
4096
  * it is literal plain text (the apostrophe in "don't"). This is exact
3372
4097
  * on valid YAML: a quote at a value position always opens. */
3373
- static int quote_opens(char prev) {
3374
- return prev == '\n' || prev == ' ' || prev == '\t' || prev == '-' || prev == ':' ||
3375
- prev == '?' || prev == ',' || prev == '[' || prev == '{' || prev == '&' || prev == '*' ||
3376
- prev == '!';
3377
- }
3378
-
3379
4098
  /* Advances quote/flow/comment state over buf[scanned, len) and returns
3380
4099
  * the offset of the LAST safe cut (0 = none). When the final bytes'
3381
4100
  * meaning depends on bytes not yet fed (a quote that may be doubled,
@@ -3387,10 +4106,35 @@ static size_t step_scan(yep_stepstate* st) {
3387
4106
  size_t hold = st->len;
3388
4107
  size_t i = st->scanned;
3389
4108
  for (; i < st->len; i++) {
3390
- char c = p[i];
4109
+ unsigned char c = (unsigned char)p[i];
3391
4110
  if (st->in_comment) {
4111
+ /* the engine normalizes NEL/LS/PS to \n before the parse,
4112
+ * so they end a comment too -- and they are LINE BREAKS:
4113
+ * the line-start state must arm exactly like the main
4114
+ * path's break branches, or the post-break '#' reads as
4115
+ * content (at_break stays 0), a closer inside it closes
4116
+ * the scan's flow while the engine's stays open, and the
4117
+ * next --- cut lands inside that flow ("{\n#\u2029#]\n---
4118
+ * {..." whole 2 / chunked 3, the nightly's run
4119
+ * 39062394941) */
3392
4120
  if (c == '\n') {
3393
4121
  st->in_comment = 0;
4122
+ st->tok_start = 1;
4123
+ st->word_led = 0;
4124
+ st->at_break = 1;
4125
+ } else if (c == 0xC2 && i + 1 < st->len && (unsigned char)p[i + 1] == 0x85) {
4126
+ i++;
4127
+ st->in_comment = 0;
4128
+ st->tok_start = 1;
4129
+ st->word_led = 0;
4130
+ st->at_break = 1;
4131
+ } else if (c == 0xE2 && i + 2 < st->len && (unsigned char)p[i + 1] == 0x80 &&
4132
+ ((unsigned char)p[i + 2] & 0xFE) == 0xA8) {
4133
+ i += 2;
4134
+ st->in_comment = 0;
4135
+ st->tok_start = 1;
4136
+ st->word_led = 0;
4137
+ st->at_break = 1;
3394
4138
  }
3395
4139
  continue;
3396
4140
  }
@@ -3422,18 +4166,137 @@ static size_t step_scan(yep_stepstate* st) {
3422
4166
  }
3423
4167
  continue;
3424
4168
  }
4169
+ if (st->in_verbatim) {
4170
+ if (c == '>') {
4171
+ st->in_verbatim = 0;
4172
+ }
4173
+ continue; /* verbatim tag bytes never cut (a truncated tag
4174
+ * at the buffer end holds the tail whole) */
4175
+ }
3425
4176
  if (c == '#') {
3426
4177
  char prev = i > 0 ? p[i - 1] : '\n';
3427
- if (prev == '\n' || prev == ' ' || prev == '\t') {
4178
+ if (prev == '\n' || prev == '\r' || prev == ' ' || prev == '\t' || st->at_break) {
3428
4179
  st->in_comment = 1;
4180
+ st->at_break = 0;
4181
+ continue;
4182
+ }
4183
+ }
4184
+ if (c == '\n' || (c == '\r' && (i + 1 >= st->len || p[i + 1] != '\n'))) {
4185
+ /* a lone \r is a line break too (the engine's break rule);
4186
+ * missing it left the line-start state desynced across the
4187
+ * cut and whole/chunked marked the flow's END differently
4188
+ * (the nightly's run 38549939326) */
4189
+ st->tok_start = 1;
4190
+ st->word_led = 0;
4191
+ st->at_break = 1;
4192
+ continue;
4193
+ }
4194
+ if (c == 0xC2) {
4195
+ /* NEL (C2 85): the engine's normalize maps it to \n before
4196
+ * the parse -- the scan must break on it too, or a comment
4197
+ * after the break goes unseen and the cut lands inside the
4198
+ * still-open flow (the nightly's round 120). C2 leads all
4199
+ * of U+0080-U+00BF: bounds FIRST (the old condition read
4200
+ * p[i+1] past the buffer on a split), hold only a prefix
4201
+ * that may still complete. */
4202
+ if (i + 1 >= st->len) {
4203
+ hold = i;
4204
+ break;
4205
+ }
4206
+ if ((unsigned char)p[i + 1] == 0x85) {
4207
+ i++;
4208
+ st->tok_start = 1;
4209
+ st->word_led = 0;
4210
+ st->at_break = 1;
4211
+ continue;
4212
+ }
4213
+ }
4214
+ if (c == 0xE2) {
4215
+ /* LS/PS (E2 80 A8 / A9): the same normalize law. E2 also
4216
+ * leads U+2000-U+2FFF symbols, so hold only a prefix that
4217
+ * may still complete into a break across the chunk
4218
+ * boundary (byte-by-byte feeding splits it every time). */
4219
+ if (i + 1 >= st->len) {
4220
+ hold = i;
4221
+ break;
4222
+ }
4223
+ if ((unsigned char)p[i + 1] != 0x80) {
4224
+ /* not the LS/PS family: plain content */
4225
+ } else if (i + 2 >= st->len) {
4226
+ hold = i;
4227
+ break;
4228
+ } else if (((unsigned char)p[i + 2] & 0xFE) == 0xA8) {
4229
+ i += 2;
4230
+ st->tok_start = 1;
4231
+ st->word_led = 0;
4232
+ st->at_break = 1;
3429
4233
  continue;
3430
4234
  }
3431
4235
  }
3432
- if ((c == '\'' || c == '"') && quote_opens(i > 0 ? p[i - 1] : '\n')) {
3433
- st->quote = c;
4236
+ if (c == ' ' || c == '\t') {
4237
+ /* in a flow a space always ends the token: the next byte
4238
+ * may begin the VALUE ("{f 'x'}" — the quote after the
4239
+ * plain key is the value's opener, and missing it let the
4240
+ * cut land inside the quoted span, splitting the scalar
4241
+ * across the chunk boundary: whole saw the far close and
4242
+ * ate the '---' marker, chunked errored on the unterminated
4243
+ * prefix — the nightly's run 38534965021) */
4244
+ st->tok_start = (st->in_flow > 0) ? 1 : st->word_led;
3434
4245
  continue;
3435
4246
  }
4247
+ if (st->tok_start) {
4248
+ st->at_break = 0; /* any content byte ends the line-start
4249
+ * state ("x#": the '#' is content) */
4250
+ if (c == '!') {
4251
+ if (i + 1 >= st->len) {
4252
+ hold = i; /* a possibly-truncated '!<': the next
4253
+ * byte decides verbatim vs named */
4254
+ break;
4255
+ }
4256
+ if (p[i + 1] == '<') {
4257
+ st->in_verbatim = 1;
4258
+ i++; /* the '<' rides */
4259
+ continue;
4260
+ }
4261
+ }
4262
+ st->word_led = (c == '&' || c == '*' || c == '!' || c == '-' || c == '?' || c == ':' ||
4263
+ c == ',' || c == '[' || c == '{' || c == ']' || c == '}');
4264
+ if (c == '?' && st->in_flow > 0) {
4265
+ /* a flow '?' opens the explicit-key MAP: the engine is
4266
+ * inside one more collection than the bracket count
4267
+ * shows, so marker cuts must stay out of it. The map
4268
+ * closes only at a closer (or never, on garbage), so
4269
+ * this over-counts on "[? a : b, c]" — the safe
4270
+ * direction (a missed cut degrades toward whole-parse
4271
+ * identity; the nightly's crash-c2894d7e, run
4272
+ * 39091724384, cut at a --- the engine read as flow
4273
+ * content and the lone '-' at the chunk edge errored
4274
+ * differently than the whole parse). */
4275
+ st->in_flow++;
4276
+ }
4277
+ if (c == '\'' || c == '"') {
4278
+ st->quote = c; /* a quote at a word start IS a quoted token */
4279
+ st->tok_start = 0;
4280
+ continue;
4281
+ }
4282
+ st->tok_start = 0;
4283
+ }
4284
+ if (c == ',' || c == ':' || c == '[' || c == '{' || c == ']' || c == '}') {
4285
+ st->tok_start = 1; /* flow separators: the next byte may begin a
4286
+ * token with no space ("{\"a\":...") */
4287
+ }
3436
4288
  if (c == '[' || c == '{') {
4289
+ /* EVERY opener counts -- the scan cannot tell a flow
4290
+ * opener from a block plain's content brace: "k: {", "&b["
4291
+ * and "[\n---" are real opens, "U{" is not, and no local
4292
+ * state separates them (the nightly's rounds 109-112 went
4293
+ * through unconditional counting, tok_start, and
4294
+ * tok_start+word_led). Over-counting only SUPPRESSES cuts
4295
+ * (the marker logic below stays out of flow), and a missed
4296
+ * cut degrades toward whole-parse semantics -- the
4297
+ * identity invariant's safe direction. The directive law
4298
+ * does not depend on this branch: the '%' case is hoisted
4299
+ * below. */
3437
4300
  st->in_flow++;
3438
4301
  continue;
3439
4302
  }
@@ -3443,13 +4306,97 @@ static size_t step_scan(yep_stepstate* st) {
3443
4306
  }
3444
4307
  continue;
3445
4308
  }
4309
+ if (c == '%') {
4310
+ /* hoisted out of the block branch: a directive line must
4311
+ * pair with its --- even when brace over-counting keeps
4312
+ * the scan "in flow" (a content brace never closed) --
4313
+ * the marker cut below lives inside the block branch and
4314
+ * would otherwise land between the directive and its ---
4315
+ * (the nightly's round 109) */
4316
+ st->saw_directive = 1; /* must stay with its --- line */
4317
+ continue;
4318
+ }
3446
4319
  if (st->in_flow == 0) {
3447
4320
  char prev = i > 0 ? p[i - 1] : '\n';
3448
- if (prev != '\n') {
4321
+ if (prev != '\n' && prev != '\r') {
3449
4322
  continue;
3450
4323
  }
3451
- if (c == '%') {
3452
- st->saw_directive = 1; /* must stay with its --- line */
4324
+ /* Block scalars: once a '>' / '|' header is seen, its
4325
+ * content is literal and the fold reader's extent depends
4326
+ * on bytes this run may not hold (markers, comments and
4327
+ * blank lines all ride along to EOF in the lenient
4328
+ * grammar) — so consume every line without updating the
4329
+ * quote/flow/comment state and WITHOUT cutting: a split
4330
+ * inside the fold changes the scalar's value (the
4331
+ * nightly's run 38291177166, ">\n---\n" and friends).
4332
+ * The fold ends only at the reader's own rule: a dedented
4333
+ * line that is not blank, not a comment, not a marker. */
4334
+ if (st->bs_open) {
4335
+ size_t k = i;
4336
+ uint32_t ind = 0;
4337
+ while (k < st->len && p[k] == ' ') {
4338
+ ind++;
4339
+ k++;
4340
+ }
4341
+ if (k >= st->len) {
4342
+ hold = i; /* an incomplete line */
4343
+ break;
4344
+ }
4345
+ if (p[k] == '\n' || p[k] == '#') {
4346
+ size_t e = k;
4347
+ while (e < st->len && p[e] != '\n') {
4348
+ e++;
4349
+ }
4350
+ if (e >= st->len) {
4351
+ hold = i;
4352
+ break;
4353
+ }
4354
+ i = e; /* blank or comment content: the loop's i++ crosses the break */
4355
+ continue;
4356
+ }
4357
+ int marker =
4358
+ (p[k] == '-' && k + 2 < st->len && p[k + 1] == '-' && p[k + 2] == '-') ||
4359
+ (p[k] == '.' && k + 2 < st->len && p[k + 1] == '.' && p[k + 2] == '.');
4360
+ if ((p[k] == '-' || p[k] == '.') && k + 2 >= st->len) {
4361
+ hold = i; /* a marker possibly truncated by the chunk */
4362
+ break;
4363
+ }
4364
+ if (!marker && st->bs_indent != (uint32_t)-1 && ind <= st->bs_indent) {
4365
+ st->bs_open = 0; /* the fold ends here: normal processing resumes */
4366
+ } else {
4367
+ size_t e = k;
4368
+ while (e < st->len && p[e] != '\n') {
4369
+ e++;
4370
+ }
4371
+ if (e >= st->len) {
4372
+ hold = i;
4373
+ break;
4374
+ }
4375
+ i = e;
4376
+ continue;
4377
+ }
4378
+ }
4379
+ if ((c == '>' || c == '|') &&
4380
+ (i + 1 >= st->len || p[i + 1] == ' ' || p[i + 1] == '\t' || p[i + 1] == '\n' ||
4381
+ p[i + 1] == '\r' || (p[i + 1] >= '0' && p[i + 1] <= '9') || p[i + 1] == '+' ||
4382
+ p[i + 1] == '-')) {
4383
+ uint32_t ind = 0;
4384
+ size_t k = i;
4385
+ while (k > 0 && p[k - 1] == ' ') {
4386
+ k--;
4387
+ ind++;
4388
+ }
4389
+ st->bs_open = 1;
4390
+ st->bs_indent = ind;
4391
+ size_t e = i;
4392
+ while (e < st->len && p[e] != '\n') {
4393
+ e++;
4394
+ }
4395
+ if (e >= st->len) {
4396
+ hold = i; /* an incomplete header line */
4397
+ break;
4398
+ }
4399
+ i = e;
3453
4400
  continue;
3454
4401
  }
3455
4402
  if (c == '-') {
@@ -3460,7 +4407,26 @@ static size_t step_scan(yep_stepstate* st) {
3460
4407
  if (p[i + 1] == '-' && p[i + 2] == '-') {
3461
4408
  char after = p[i + 3];
3462
4409
  if (after == '\n' || after == ' ' || after == '\t' || after == '\r') {
3463
- if (st->saw_directive == 0 && i > 0) {
4410
+ /* a doc-inline block scalar ("--- |"): the node
4411
+ * hangs off the DOCUMENT, so its fold follows
4412
+ * the root rule — only a later marker ends it
4413
+ * (col-0 lines are content). The line-start
4414
+ * header check below cannot see it (the '|'
4415
+ * rides mid-line after the marker), and
4416
+ * without the state the fold's content
4417
+ * poisons the quote/flow tracking (the
4418
+ * nightly's run 38352718743). */
4419
+ if (!st->bs_open) {
4420
+ size_t k2 = i + 3;
4421
+ while (k2 < st->len && (p[k2] == ' ' || p[k2] == '\t')) {
4422
+ k2++;
4423
+ }
4424
+ if (k2 < st->len && (p[k2] == '>' || p[k2] == '|')) {
4425
+ st->bs_open = 1;
4426
+ st->bs_indent = (uint32_t)-1; /* root rule: markers only */
4427
+ }
4428
+ }
4429
+ if (st->saw_directive == 0 && i > 0 && !st->bs_open) {
3464
4430
  last_cut = i; /* interior document boundary */
3465
4431
  }
3466
4432
  st->saw_directive = 0; /* this --- owns the directives */
@@ -3484,6 +4450,7 @@ int yep_engine_step(yep_engine* e, const char* chunk, size_t len, int final, con
3484
4450
  }
3485
4451
  }
3486
4452
  yep_stepstate* st = e->step;
4453
+ st->tok_start = 1; /* the buffer starts at a line start */
3487
4454
  if (len > 0) {
3488
4455
  if (st->len + len > st->cap) {
3489
4456
  size_t cap = st->cap ? st->cap : 4096;
@@ -3518,6 +4485,7 @@ int yep_engine_step(yep_engine* e, const char* chunk, size_t len, int final, con
3518
4485
  /* the tail began at a --- line in a clean state */
3519
4486
  st->in_flow = 0;
3520
4487
  st->quote = 0;
4488
+ st->in_verbatim = 0;
3521
4489
  st->in_comment = 0;
3522
4490
  st->saw_directive = 0;
3523
4491
  return 0;
@@ -3539,8 +4507,17 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3539
4507
  }
3540
4508
  yep_free(e->sys, e->norm_buf);
3541
4509
  e->norm_buf = NULL;
3542
- e->p = e_normalize_breaks(e, buf, len);
3543
- e->len = (e->norm_buf != NULL) ? strlen(e->norm_buf) : len;
4510
+ /* enc_buf is NOT reset here: yep_engine_run's front-end may have
4511
+ * transcoded THIS call's input into it microseconds ago -- buf
4512
+ * points into it, and freeing it before the break-normalize scan
4513
+ * was a use-after-free on every UTF-16/32 input (ASan: the 4-byte
4514
+ * artifact "ff fe fe fe", the nightly's combined-build round).
4515
+ * It is run-scoped scratch owned by the front-end; destroy frees
4516
+ * it, and the next transcode replaces it. */
4517
+ e->parsing_final = emit_end;
4518
+ size_t norm_len = len;
4519
+ e->p = e_normalize_breaks(e, buf, len, &norm_len);
4520
+ e->len = norm_len;
3544
4521
  e->pos = 0;
3545
4522
  e->line = 1 + e->line_base;
3546
4523
  e->line_base = 0;
@@ -3550,12 +4527,17 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3550
4527
  e->facts_cache_line = 0;
3551
4528
  e->depth = 0;
3552
4529
  yep_nametab_clear(&e->anchors);
4530
+ if (e->anchor_placed != NULL) {
4531
+ memset(e->anchor_placed, 0, e->anchor_placed_cap);
4532
+ }
4533
+ e->alias_memo_len = 0;
3553
4534
  e->tagmap_n = 0;
3554
4535
  e->saw_yaml = 0;
3555
4536
  e->doc_begin = (size_t)-1;
3556
4537
  e->doc_content = 0;
3557
4538
  e->q_key_pending = 0;
3558
4539
  e->q_value_pending = 0;
4540
+ e->qmd_n = 0;
3559
4541
  e->sink = sink;
3560
4542
  yep_error_clear(&e->err);
3561
4543
  if (e->resolver == NULL) {
@@ -3603,6 +4585,26 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3603
4585
  e_fail(e, YEP_ERR_BAD_DIRECTIVE, e->pos); /* repeated %YAML */
3604
4586
  goto fail;
3605
4587
  }
4588
+ {
4589
+ /* libyaml: the directive name is [0-9A-Za-z_-]+
4590
+ * followed by a blank or the line end -- an empty or
4591
+ * malformed name is an error ("could not find expected
4592
+ * directive name"). The lenient skip let "%]" ride as a
4593
+ * legal directive line whose DOCUMENT_START inclusion
4594
+ * diverged across the step boundary (the nightly's
4595
+ * round 117) */
4596
+ size_t n0 = li.offset + li.indent + 1;
4597
+ while (n0 < li.end && (yep_ct_is((unsigned char)e->p[n0], YEP_CT_ALPHA) ||
4598
+ yep_ct_is((unsigned char)e->p[n0], YEP_CT_DIGIT) ||
4599
+ e->p[n0] == '_' || e->p[n0] == '-')) {
4600
+ n0++;
4601
+ }
4602
+ if (n0 == li.offset + li.indent + 1 ||
4603
+ (n0 < li.end && e->p[n0] != ' ' && e->p[n0] != '\t')) {
4604
+ e_fail(e, YEP_ERR_BAD_DIRECTIVE, li.offset + li.indent + 1);
4605
+ goto fail;
4606
+ }
4607
+ }
3606
4608
  {
3607
4609
  /* %YAML takes exactly one version argument */
3608
4610
  size_t chk = li.offset + li.indent + 1;
@@ -3644,16 +4646,31 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3644
4646
  h0++;
3645
4647
  }
3646
4648
  size_t h1 = h0;
3647
- while (h1 < li.end && e->p[h1] != ' ') {
4649
+ while (h1 < li.end && e->p[h1] != ' ' && e->p[h1] != '\t') {
3648
4650
  h1++;
3649
4651
  }
3650
4652
  size_t p0 = h1;
3651
- while (p0 < li.end && e->p[p0] == ' ') {
4653
+ while (p0 < li.end && (e->p[p0] == ' ' || e->p[p0] == '\t')) {
3652
4654
  p0++;
3653
4655
  }
3654
- size_t p1 = li.end;
3655
- while (p1 > p0 && (e->p[p1 - 1] == ' ' || e->p[p1 - 1] == '\r')) {
3656
- p1--;
4656
+ size_t p1 = p0;
4657
+ while (p1 < li.end && e->p[p1] != ' ' && e->p[p1] != '\t' && e->p[p1] != '#') {
4658
+ p1++;
4659
+ }
4660
+ {
4661
+ /* libyaml: only blanks and an optional comment may
4662
+ * follow the prefix ("did not find expected comment
4663
+ * or line break") — trailing content riding the
4664
+ * prefix made parseable tags that never re-parsed
4665
+ * (the nightly's "%TAG !m! !mymy 3 ..." trap) */
4666
+ size_t r = p1;
4667
+ while (r < li.end && (e->p[r] == ' ' || e->p[r] == '\t')) {
4668
+ r++;
4669
+ }
4670
+ if (r < li.end && e->p[r] != '#') {
4671
+ e_fail(e, YEP_ERR_BAD_DIRECTIVE, r);
4672
+ return -1;
4673
+ }
3657
4674
  }
3658
4675
  if (h1 > h0 && p1 > p0 && e->p[h0] == '!') {
3659
4676
  e->tagmap[e->tagmap_n].handle.p = e->p + h0;
@@ -3698,8 +4715,16 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3698
4715
  return -2;
3699
4716
  }
3700
4717
  doc_open = 0;
3701
- e->tagmap_n = 0; /* directives are per-document */
3702
- yep_nametab_clear(&e->anchors); /* so are anchors */
4718
+ e->tagmap_n = 0; /* directives are per-document */
4719
+ yep_nametab_clear(&e->anchors);
4720
+ if (e->anchor_placed != NULL) {
4721
+ memset(e->anchor_placed, 0, e->anchor_placed_cap);
4722
+ }
4723
+ e->alias_memo_len = 0; /* so are anchors */
4724
+ e->alias_memo_len = 0; /* and the repeat-alias memo: doc 1's
4725
+ * "*ORIGIN" memoized the ordinal and doc 2's
4726
+ * identical name answered from it ACROSS the
4727
+ * boundary (the nightly's run 38266487549) */
3703
4728
  }
3704
4729
  e_event_init(&ev, YEP_EV_DOCUMENT_START);
3705
4730
  ev.style = 1; /* explicit marker */
@@ -3713,6 +4738,14 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3713
4738
  doc_open = 1;
3714
4739
  e->saw_yaml = 0;
3715
4740
  e->last_root_flow = 0; /* a new document may carry its own directives */
4741
+ /* the '?' / ':' pair state is per-document too: doc 4's flow
4742
+ * set q_key_pending and the boundary left it set, so doc 5's
4743
+ * bare ':' skipped its empty-key event — whole-buffer lost a
4744
+ * record the chunked feed (per-run reset) emitted (the
4745
+ * nightly's run 38287390219) */
4746
+ e->q_key_pending = 0;
4747
+ e->q_value_pending = 0;
4748
+ e->qmd_n = 0;
3716
4749
  e->pos = li.offset + li.indent + 3;
3717
4750
  e_skip_inline_space(e);
3718
4751
  if (!e_at_eol(e)) {
@@ -3732,12 +4765,18 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3732
4765
  }
3733
4766
  if (li.flags & YEP_LF_DOC_END) {
3734
4767
  {
4768
+ /* the "..." marker's line ends at a break — a lone '\r'
4769
+ * included: skipping it as whitespace read the NEXT
4770
+ * line's "---" as trailing junk on the marker's own line
4771
+ * ("...\r---\n" errored whole-buffer while the chunked
4772
+ * feed's boundary split the lines correctly — the
4773
+ * nightly's round 105) */
3735
4774
  size_t rest = li.offset + li.indent + 3;
3736
- while (rest < e->len &&
3737
- (e->p[rest] == ' ' || e->p[rest] == '\t' || e->p[rest] == '\r')) {
4775
+ while (rest < e->len && (e->p[rest] == ' ' || e->p[rest] == '\t')) {
3738
4776
  rest++;
3739
4777
  }
3740
- if (rest < e->len && e->p[rest] != '\n' && e->p[rest] != '#') {
4778
+ if (rest < e->len && e->p[rest] != '\n' && e->p[rest] != '\r' &&
4779
+ e->p[rest] != '#') {
3741
4780
  e_fail(e, YEP_ERR_UNEXPECTED, rest);
3742
4781
  goto fail;
3743
4782
  }
@@ -3760,6 +4799,10 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3760
4799
  e_pos_mark(e, li.offset + li.indent + 3, &ev.end_line, &ev.end_col);
3761
4800
  e->tagmap_n = 0; /* directives are per-document */
3762
4801
  yep_nametab_clear(&e->anchors);
4802
+ if (e->anchor_placed != NULL) {
4803
+ memset(e->anchor_placed, 0, e->anchor_placed_cap);
4804
+ }
4805
+ e->alias_memo_len = 0;
3763
4806
  if (emit_now(e, &ev) != 0) {
3764
4807
  return -2;
3765
4808
  }
@@ -3862,18 +4905,33 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3862
4905
  goto fail;
3863
4906
  }
3864
4907
  }
3865
- if (!e->q_key_pending) {
4908
+ /* this ':' line completes the pending explicit pair only at
4909
+ * the pending pair's own map depth — deeper inside the key's
4910
+ * own content it is a plain bare-colon pair, and consuming
4911
+ * the pending there skipped this line's empty-key emit,
4912
+ * leaving an orphaned (anchorable) child the DOM could not
4913
+ * pair and the emit dropped (the nightly's round 102) */
4914
+ int completes_q =
4915
+ e->q_key_pending && e->qmd_n > 0 && e->depth == e->qmd_stack[e->qmd_n - 1];
4916
+ if (!completes_q) {
3866
4917
  yep_event kv;
3867
4918
  e_event_init(&kv, YEP_EV_SCALAR);
3868
4919
  kv.style = YEP_STYLE_PLAIN;
3869
4920
  kv.implicit = 1;
4921
+ kv.line = e->line;
4922
+ kv.col = (uint32_t)e_col(e, e->pos) + 1;
3870
4923
  if (emit_now(e, &kv) != 0) {
3871
4924
  return -2;
3872
4925
  }
4926
+ } else {
4927
+ e->q_key_pending = 0;
4928
+ e->q_value_pending = 0;
4929
+ if (e->qmd_n > 0) {
4930
+ e->qmd_n--; /* the ':' line supplied the innermost value */
4931
+ }
3873
4932
  }
3874
- e->q_key_pending = 0;
3875
- e->q_value_pending = 0;
3876
4933
  e->pos++;
4934
+ e->cv_line = e->line; /* the value parses on THIS line's colon */
3877
4935
  rc = e_parse_value(e, YEP_CTX_AFTER_DASH, e->frames[e->depth - 1].col);
3878
4936
  if (rc != 0) {
3879
4937
  if (rc == -2) {
@@ -3959,6 +5017,10 @@ static int engine_run_impl(yep_engine* e, const char* buf, size_t len, const yep
3959
5017
  }
3960
5018
  e->tagmap_n = 0;
3961
5019
  yep_nametab_clear(&e->anchors);
5020
+ if (e->anchor_placed != NULL) {
5021
+ memset(e->anchor_placed, 0, e->anchor_placed_cap);
5022
+ }
5023
+ e->alias_memo_len = 0;
3962
5024
  }
3963
5025
  if (emit_end) {
3964
5026
  e_event_init(&ev, YEP_EV_STREAM_END);