nomen-lang 0.6.0 → 0.6.2

package/core/System/Text/Regex.nm

@@ -255,12 +255,13 @@ func fold_pattern = (string pattern, move out string) {
 	while i < n; i += 1 {
 		var char c = pattern.at(i)
 		if (c as int) == 92 {
-			// Escaped char: shorthands and backrefs keep their backslash;
-			// other escaped letters become both-case classes; escaped
-			// punctuation is copied literally.
+			// Escaped char: shorthands, backrefs, and control escapes
+			// (\n \r \t) keep their backslash; other escaped letters
+			// become both-case classes; escaped punctuation is copied
+			// literally.
 			if i + 1 < n {
 				var char e = pattern.at(i + 1)
-				if is_shorthand(e) || is_backref_digit(e) {
+				if is_shorthand(e) || is_backref_digit(e) || is_control_escape(e) {
 					sb.append_char(c)
 					sb.append_char(e)
 				} else {
@@ -317,9 +318,10 @@ func fold_class = (string pattern, int cs, int ce, ref StringBuilder dst) {
 			if i + 1 < content_end {
 				var char e = pattern.at(i + 1)
 				dst.append_char(e)
-				// A shorthand keeps its backslash; other escaped letters
-				// gain their case counterpart.
-				if is_alpha_byte(e) && !is_shorthand(e) {
+				// A shorthand or control escape (\n \r \t) keeps its
+				// backslash; other escaped letters gain their case
+				// counterpart.
+				if is_alpha_byte(e) && !is_shorthand(e) && !is_control_escape(e) {
 					dst.append_char(swap_case_byte(e))
 				}
 				i += 1
@@ -469,18 +471,20 @@ func collect_first_bytes = (
 //
 // Handles the shapes the engine's matcher can produce: a leading anchor is
 // zero-width (skipped), a leading group contributes its CONTENT's first bytes
-// (recursively over the group's own top-level alternatives), and a group
-// quantified with '?'/'*' is nullable — what FOLLOWS it can also be first, so
-// the scan continues past it. A wildcard '.' or an empty alternative makes the
-// set indeterminable.
+// (recursively over the group's own top-level alternatives), and ANY leading
+// atom — literal, escaped, class, or group — quantified with '?'/'*' is
+// nullable: what FOLLOWS it can also be first, so the scan continues past it.
+// A wildcard '.' or an empty alternative makes the set indeterminable.
 func alt_first_bytes = (string pattern, int start, int end, ref StringBuilder sb, out int) {
 	var int i = start
 	while i < end {
 		var char c = pattern.at(i)
 		if (c as int) == 92 {
-			// escaped literal: first byte is the escaped char — unless it's
-			// a backreference (depends on a capture) or a complement
-			// shorthand (not an inclusion set), which disable the prefilter.
+			// escaped literal: first byte is the byte the escape names
+			// (\n is LF, \r is CR, \t is TAB; any other escaped character
+			// is its own literal byte) — unless it's a backreference
+			// (depends on a capture) or a complement shorthand (not an
+			// inclusion set), which disable the prefilter.
 			if i + 1 < end {
 				var char e = pattern.at(i + 1)
 				if is_backref_digit(e) {
@@ -490,9 +494,17 @@ func alt_first_bytes = (string pattern, int start, int end, ref StringBuilder sb
 					if shorthand_members(e, ref sb) < 0 {
 						return -1
 					}
+				} else {
+					sb.append_char(escaped_byte_value(e) as char)
+				}
+				// A '?'/'*' quantifier makes the atom nullable — the first
+				// byte can also come from whatever follows it.
+				var int cont = nullable_quant_after(pattern, i + 2, end)
+				if cont < 0 {
 					return 0
 				}
-				sb.append_char(e)
+				i = cont
+				continue
 			}
 			return 0
 		}
@@ -509,7 +521,12 @@ func alt_first_bytes = (string pattern, int start, int end, ref StringBuilder sb
 				}
 			}
 			add_class_chars(pattern, i + 1, a_end - 1, ref sb)
-			return 0
+			var int cont = nullable_quant_after(pattern, a_end, end)
+			if cont < 0 {
+				return 0
+			}
+			i = cont
+			continue
 		}
 		if c == '(' {
 			var int a_end = atom_end_pos(pattern, i, end)
@@ -518,27 +535,11 @@ func alt_first_bytes = (string pattern, int start, int end, ref StringBuilder sb
 			if collect_first_bytes(pattern, i + 1, a_end - 1, ref sb) < 0 {
 				return -1
 			}
-			// A '?'/'*' quantifier makes the group nullable — the first byte
-			// can also come from whatever follows it. A second '?' marks the
-			// quantifier lazy and is skipped either way.
-			var int after = a_end
-			var nullable = 0
-			if after < end {
-				var char q = pattern.at(after)
-				if q == '?' || q == '*' {
-					nullable = 1
-					after += 1
-					if after < end {
-						if pattern.at(after) == '?' {
-							after += 1
-						}
-					}
-				}
-			}
-			if nullable == 0 {
+			var int cont = nullable_quant_after(pattern, a_end, end)
+			if cont < 0 {
 				return 0
 			}
-			i = after
+			i = cont
 			continue
 		}
 		if c == '^' || c == '$' {
@@ -552,9 +553,40 @@ func alt_first_bytes = (string pattern, int start, int end, ref StringBuilder sb
 			return -1
 		}
 		sb.append_char(c)
-		return 0
+		var int cont = nullable_quant_after(pattern, i + 1, end)
+		if cont < 0 {
+			return 0
+		}
+		i = cont
+		continue
+	}
+	// An empty alternative (nothing left after nullable atoms — or nothing at
+	// all) can match at any position — indeterminable.
+	return -1
+}
+
+// If the atom ending at `after` carries a nullable quantifier ('?' or '*',
+// optionally marked lazy by a second '?'), return the position just past the
+// quantifier; otherwise return -1. Callers keep scanning from the returned
+// position: a nullable leading atom means the alternative's match can also
+// START with whatever follows, so those bytes belong in the first-byte set
+// too. ('+' is NOT nullable — the atom must match once — so it leaves the
+// atom's own bytes as the whole first set.)
+func nullable_quant_after = (string pattern, int after, int end, out int) {
+	var int j = after
+	if j >= end {
+		return -1
+	}
+	var char q = pattern.at(j)
+	if q == '?' || q == '*' {
+		j += 1
+		if j < end {
+			if pattern.at(j) == '?' {
+				j += 1
+			}
+		}
+		return j
 	}
-	// An empty alternative can match at any position — indeterminable.
 	return -1
 }
 
@@ -569,22 +601,34 @@ func add_class_chars = (string pattern, int start, int end, ref StringBuilder sb
 	}
 	var int i = cs
 	while i < end {
-		if i + 2 < end {
-			if pattern.at(i + 1) == '-' {
-				var int lo = pattern.at(i) as int
-				var int hi = pattern.at(i + 2) as int
-				while lo <= hi {
-					sb.append_char(lo as char)
-					lo += 1
+		var char c = pattern.at(i)
+		if (c as int) == 92 {
+			// Escaped byte: contribute the byte the escape names (\n is
+			// LF, \r is CR, \t is TAB; any other escaped character is its
+			// own literal byte) so the prefilter stays a superset of the
+			// matcher's first bytes.
+			if i + 1 < end {
+				sb.append_char(escaped_byte_value(pattern.at(i + 1)) as char)
+			}
+			i += 2
+		} else {
+			if i + 2 < end {
+				if pattern.at(i + 1) == '-' {
+					var int lo = pattern.at(i) as int
+					var int hi = pattern.at(i + 2) as int
+					while lo <= hi {
+						sb.append_char(lo as char)
+						lo += 1
+					}
+					i += 3
+				} else {
+					sb.append_char(c)
+					i += 1
 				}
-				i += 3
 			} else {
-				sb.append_char(pattern.at(i))
+				sb.append_char(c)
 				i += 1
 			}
-		} else {
-			sb.append_char(pattern.at(i))
-			i += 1
 		}
 	}
 }
@@ -732,7 +776,23 @@ func match_here = (
 		}
 	}
 
-	// No quantifier: match the atom once, then the rest of the pattern.
+	// No quantifier: match the atom once, then the rest of the pattern. A
+	// GROUP atom goes through the continuation-aware path: its inner
+	// alternatives retry when the rest fails on an earlier one (JS-style
+	// backtracking — `(head|header)(\s|$|>)` must still match "header>").
+	if pc == '(' {
+		return match_group_then_rest(
+			pattern,
+			pat_pos,
+			a_end,
+			a_end,
+			pat_end,
+			input,
+			in_pos,
+			in_len,
+			ref caps,
+		)
+	}
 	var int next = match_single_atom(pattern, pat_pos, a_end, input, in_pos, in_len, ref caps)
 	if next < 0 {
 		return -1
@@ -740,6 +800,97 @@ func match_here = (
 	return match_here(pattern, a_end, pat_end, input, next, in_len, ref caps)
 }
 
+// The position of the first top-level '|' in [start, end), or -1. Escapes,
+// character classes, and nested groups are skipped so only depth-0
+// alternation operators count (same scanning discipline as match_here).
+func next_top_level_alt = (string pattern, int start, int end, out int) {
+	var depth = 0
+	var int i = start
+	while i < end {
+		var char c = pattern.at(i)
+		if (c as int) == 92 {
+			i += 2
+		} else {
+			if c == '[' {
+				i = atom_end_pos(pattern, i, end)
+			} else {
+				if c == '(' {
+					depth += 1
+				}
+				if c == ')' {
+					if depth > 0 {
+						depth -= 1
+					}
+				}
+				if (c as int) == 124 {
+					if depth == 0 {
+						return i
+					}
+				}
+				i += 1
+			}
+		}
+	}
+	return -1
+}
+
+// Match the group atom spanning [atom_start, atom_end) ('(' at atom_start)
+// and then the continuation [rest, pat_end), retrying the group's LATER
+// alternatives whenever the continuation fails on an earlier one. Without
+// this, a group commits to the first alternative that matches the group
+// ALONE and a failing continuation aborts the whole match — the engine had
+// no backtracking into a group's alternatives (PORT.md: `(head|header)`
+// never reached `header` when `(\s|$|>)` rejected what `head` left behind).
+// A group without a top-level '|' behaves exactly like the old
+// match-atom-then-continue path, and a failed trial restores the group's
+// previous capture span before the next attempt.
+func match_group_then_rest = (
+	string pattern,
+	int atom_start,
+	int atom_end,
+	int rest,
+	int pat_end,
+	string input,
+	int in_pos,
+	int in_len,
+	ref List<int> caps,
+	out int
+) {
+	var int g = group_index_at(pattern, atom_start)
+	var int at = last_capture_index(caps, g)
+	var int old_start = -1
+	var int old_end = -1
+	if at >= 0 {
+		old_start = caps.at(at + 1)
+		old_end = caps.at(at + 2)
+	}
+	var int inner_start = atom_start + 1
+	var int inner_end = atom_end - 1
+	var int s = inner_start
+	while s <= inner_end {
+		var int bar = next_top_level_alt(pattern, s, inner_end)
+		var int e = inner_end
+		if bar >= 0 {
+			e = bar
+		}
+		var int next = match_here(pattern, s, e, input, in_pos, in_len, ref caps)
+		if next >= 0 {
+			record_capture(ref caps, g, in_pos, next)
+			var int r = match_here(pattern, rest, pat_end, input, next, in_len, ref caps)
+			if r >= 0 {
+				return r
+			}
+		}
+		record_capture(ref caps, g, old_start, old_end)
+		if bar >= 0 {
+			s = bar + 1
+		} else {
+			s = inner_end + 1
+		}
+	}
+	return -1
+}
+
 // Return the position just past the atom that starts at pat_pos. Handles
 // character classes [...], groups (...), and escaped characters (\x).
 func atom_end_pos = (string pattern, int pat_pos, int pat_end, out int) {
@@ -860,7 +1011,7 @@ func match_single_atom = (
 				return -1
 			}
 			if in_pos < in_len {
-				if input.at(in_pos) == ec {
+				if (input.at(in_pos) as int) == escaped_byte_value(ec) {
 					return in_pos + 1
 				}
 			}
@@ -908,7 +1059,7 @@ func char_matches_atom = (
 			if is_shorthand(ec) {
 				return matches_shorthand(ec, ic)
 			}
-			return ic == ec
+			return (ic as int) == escaped_byte_value(ec)
 		}
 		return false
 	}
@@ -1018,29 +1169,99 @@ func match_star_group = (
 ) {
 	if lazy > 0 {
 		// Lazy: stop repeating first (minimum met), then try one more
-		// iteration before giving up.
+		// iteration before giving up. The iteration retries the group's
+		// top-level alternatives (see match_group_iteration).
 		if min_count <= 0 {
 			var int stop = match_here(pattern, rest, pat_end, input, in_pos, in_len, ref caps)
 			if stop >= 0 {
 				return stop
 			}
 		}
-		var int next = match_single_atom(
+		return match_group_iteration(
 			pattern,
 			atom_start,
 			atom_end,
+			rest,
+			pat_end,
 			input,
 			in_pos,
+			min_count,
+			lazy,
 			in_len,
 			ref caps,
 		)
+	}
+	// Try matching the group one more time (greedy) — each top-level
+	// alternative in turn — before stopping.
+	var int r = match_group_iteration(
+		pattern,
+		atom_start,
+		atom_end,
+		rest,
+		pat_end,
+		input,
+		in_pos,
+		min_count,
+		lazy,
+		in_len,
+		ref caps,
+	)
+	if r >= 0 {
+		return r
+	}
+	// Stop repeating and try the rest of the pattern, if the minimum is met.
+	if min_count <= 0 {
+		return match_here(pattern, rest, pat_end, input, in_pos, in_len, ref caps)
+	}
+	return -1
+}
+
+// Try ONE further iteration of a quantified group at in_pos, retrying the
+// group's top-level alternatives while the remaining star recursion fails.
+// Returns the recursion's result, or -1 when no alternative leads to a
+// match. An alternative that matches the empty string is not recursed into
+// (an empty iteration cannot make progress; the caller's stop path covers
+// it), and every failed trial restores the group's previous capture span.
+func match_group_iteration = (
+	string pattern,
+	int atom_start,
+	int atom_end,
+	int rest,
+	int pat_end,
+	string input,
+	int in_pos,
+	int min_count,
+	int lazy,
+	int in_len,
+	ref List<int> caps,
+	out int
+) {
+	var int g = group_index_at(pattern, atom_start)
+	var int at = last_capture_index(caps, g)
+	var int old_start = -1
+	var int old_end = -1
+	if at >= 0 {
+		old_start = caps.at(at + 1)
+		old_end = caps.at(at + 2)
+	}
+	var int inner_start = atom_start + 1
+	var int inner_end = atom_end - 1
+	var int s = inner_start
+	while s <= inner_end {
+		var int bar = next_top_level_alt(pattern, s, inner_end)
+		var int e = inner_end
+		if bar >= 0 {
+			e = bar
+		}
+		var int next = match_here(pattern, s, e, input, in_pos, in_len, ref caps)
 		if next >= 0 {
+			record_capture(ref caps, g, in_pos, next)
 			if next != in_pos {
 				var new_min = 0
 				if min_count > 0 {
 					new_min = min_count - 1
 				}
-				return match_star_group(
+				var int r = match_star_group(
 					pattern,
 					atom_start,
 					atom_end,
@@ -1053,40 +1274,18 @@ func match_star_group = (
 					in_len,
 					ref caps,
 				)
+				if r >= 0 {
+					return r
+				}
 			}
 		}
-		return -1
-	}
-	// Try matching the group one more time (greedy).
-	var int next = match_single_atom(pattern, atom_start, atom_end, input, in_pos, in_len, ref caps)
-	if next >= 0 {
-		if next != in_pos {
-			var new_min = 0
-			if min_count > 0 {
-				new_min = min_count - 1
-			}
-			var int r = match_star_group(
-				pattern,
-				atom_start,
-				atom_end,
-				rest,
-				pat_end,
-				input,
-				next,
-				new_min,
-				lazy,
-				in_len,
-				ref caps,
-			)
-			if r >= 0 {
-				return r
-			}
+		record_capture(ref caps, g, old_start, old_end)
+		if bar >= 0 {
+			s = bar + 1
+		} else {
+			s = inner_end + 1
 		}
 	}
-	// Stop repeating and try the rest of the pattern, if the minimum is met.
-	if min_count <= 0 {
-		return match_here(pattern, rest, pat_end, input, in_pos, in_len, ref caps)
-	}
 	return -1
 }
 
@@ -1110,6 +1309,21 @@ func match_optional = (
 		if skip >= 0 {
 			return skip
 		}
+		// A group atom includes continuation-aware alternative retries (see
+		// match_group_then_rest) in both moods.
+		if pattern.at(atom_start) == '(' {
+			return match_group_then_rest(
+				pattern,
+				atom_start,
+				atom_end,
+				rest,
+				pat_end,
+				input,
+				in_pos,
+				in_len,
+				ref caps,
+			)
+		}
 		var int next = match_single_atom(
 			pattern,
 			atom_start,
@@ -1124,11 +1338,28 @@ func match_optional = (
 		}
 		return -1
 	}
-	var int next = match_single_atom(pattern, atom_start, atom_end, input, in_pos, in_len, ref caps)
-	if next >= 0 {
-		var int r = match_here(pattern, rest, pat_end, input, next, in_len, ref caps)
-		if r >= 0 {
-			return r
+	if pattern.at(atom_start) == '(' {
+		var int grouped = match_group_then_rest(
+			pattern,
+			atom_start,
+			atom_end,
+			rest,
+			pat_end,
+			input,
+			in_pos,
+			in_len,
+			ref caps,
+		)
+		if grouped >= 0 {
+			return grouped
+		}
+	} else {
+		var int next = match_single_atom(pattern, atom_start, atom_end, input, in_pos, in_len, ref caps)
+		if next >= 0 {
+			var int r = match_here(pattern, rest, pat_end, input, next, in_len, ref caps)
+			if r >= 0 {
+				return r
+			}
 		}
 	}
 	return match_here(pattern, rest, pat_end, input, in_pos, in_len, ref caps)
@@ -1173,7 +1404,7 @@ func char_in_class = (
 						found = true
 					}
 				} else {
-					if ic == e {
+					if (ic as int) == escaped_byte_value(e) {
 						found = true
 					}
 				}
@@ -1428,6 +1659,29 @@ func is_shorthand = (char c, out bool) {
 	return code == 100 || code == 68 || code == 119 || code == 87 || code == 115 || code == 83
 }
 
+// Whether the escaped character c names a control-character shorthand
+// (\n \r \t) that the engine maps to its real control byte.
+func is_control_escape = (char c, out bool) {
+	var int code = c as int
+	return code == 110 || code == 114 || code == 116
+}
+
+// The byte an escaped atom names: \n is LF (10), \r is CR (13), \t is TAB
+// (9); every other escaped character is the literal byte of that character.
+func escaped_byte_value = (char c, out int) {
+	var int code = c as int
+	if code == 110 {
+		return 10
+	}
+	if code == 114 {
+		return 13
+	}
+	if code == 116 {
+		return 9
+	}
+	return code
+}
+
 // Whether byte b belongs to the shorthand class named by c. Uppercase names
 // are the complements (S/D/W match anything the lowercase set excludes).
 func matches_shorthand = (char c, char b, out bool) {