nomen-lang 0.2.2 → 0.3.0

package/core/System/Text/Utf8.nm

@@ -0,0 +1,163 @@
+// UTF-8 scalar helpers, implemented in pure Nomen.
+//
+// Strings are byte arrays; these functions decode and encode individual
+// Unicode scalar values without copying the text. Malformed sequences yield
+// U+FFFD with a byte length of 1 (each invalid byte is one replacement
+// character), so every byte offset either starts a character or is itself
+// reported as U+FFFD — walking with width_at can never stall or skip.
+
+/**
+ * A decoded character: its Unicode code point and UTF-8 byte length
+ **/
+pub struct DecodedChar {
+	var int code_point
+	var int byte_length
+}
+
+/**
+ * UTF-8 scalar helpers (`decode_at`/`width_at`/`encode`/`char_count`)
+ **/
+pub struct Utf8 {
+	// Decode the code point starting at byte_index.
+	// Malformed sequences yield U+FFFD with a byte_length of 1 (each
+	// invalid byte is one replacement character). Panics if byte_index is
+	// out of range (use Chars for end-aware iteration).
+	pub func decode_at = (view string text, int byte_index, out DecodedChar) {
+		if byte_index < 0 || byte_index >= text.length {
+			panic("byte index out of range")
+		}
+		var int b0 = text.at(byte_index) as int
+		// ASCII fast path.
+		if b0 < 0x80 {
+			return DecodedChar(b0, 1)
+		}
+		// Continuation bytes and C0/C1 never start a character.
+		if b0 < 0xC2 {
+			return DecodedChar(0xFFFD, 1)
+		}
+		if b0 < 0xE0 {
+			var int b1 = cont_byte(text, byte_index + 1, 0x80, 0xBF)
+			if b1 < 0 {
+				return DecodedChar(0xFFFD, 1)
+			}
+			return DecodedChar(((b0 & 0x1F) << 6) | (b1 & 0x3F), 2)
+		}
+		if b0 < 0xF0 {
+			// E0 forbids second bytes below A0 (overlong); ED forbids
+			// AA-BF (surrogate halves).
+			var int lo = 0x80
+			var int hi = 0xBF
+			if b0 == 0xE0 {
+				lo = 0xA0
+			}
+			if b0 == 0xED {
+				hi = 0x9F
+			}
+			var int b1 = cont_byte(text, byte_index + 1, lo, hi)
+			if b1 < 0 {
+				return DecodedChar(0xFFFD, 1)
+			}
+			var int b2 = cont_byte(text, byte_index + 2, 0x80, 0xBF)
+			if b2 < 0 {
+				return DecodedChar(0xFFFD, 1)
+			}
+			return DecodedChar(((b0 & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F), 3)
+		}
+		if b0 > 0xF4 {
+			return DecodedChar(0xFFFD, 1)
+		}
+		// F0 forbids second bytes below 90 (overlong); F4 forbids above 8F
+		// (past U+10FFFF).
+		var int lo = 0x80
+		var int hi = 0xBF
+		if b0 == 0xF0 {
+			lo = 0x90
+		}
+		if b0 == 0xF4 {
+			hi = 0x8F
+		}
+		var int b1 = cont_byte(text, byte_index + 1, lo, hi)
+		if b1 < 0 {
+			return DecodedChar(0xFFFD, 1)
+		}
+		var int b2 = cont_byte(text, byte_index + 2, 0x80, 0xBF)
+		if b2 < 0 {
+			return DecodedChar(0xFFFD, 1)
+		}
+		var int b3 = cont_byte(text, byte_index + 3, 0x80, 0xBF)
+		if b3 < 0 {
+			return DecodedChar(0xFFFD, 1)
+		}
+		return DecodedChar(
+			((b0 & 0x07) << 18) | ((b1 & 0x3F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F),
+			4,
+		)
+	}
+
+	// Byte length of the character starting at byte_index (1-4; malformed
+	// sequences report 1). Panics if byte_index is out of range.
+	pub func width_at = (view string text, int byte_index, out int) {
+		var d = Utf8.decode_at(text, byte_index)
+		return d.byte_length
+	}
+
+	// Encode a code point as UTF-8. Out-of-range values and surrogate halves
+	// encode as U+FFFD.
+	pub func encode = (int code_point, move out string) {
+		var int code = code_point
+		if code < 0 || code > 0x10FFFF {
+			code = 0xFFFD
+		}
+		if code >= 0xD800 && code <= 0xDFFF {
+			code = 0xFFFD
+		}
+		var sb = StringBuilder()
+		switch {
+			case code < 0x80 {
+				sb.append_char(code as char)
+			}
+			case code < 0x800 {
+				sb.append_char(((0xC0 | (code >> 6)) as char))
+				sb.append_char(((0x80 | (code & 0x3F)) as char))
+			}
+			case code < 0x10000 {
+				sb.append_char(((0xE0 | (code >> 12)) as char))
+				sb.append_char(((0x80 | ((code >> 6) & 0x3F)) as char))
+				sb.append_char(((0x80 | (code & 0x3F)) as char))
+			}
+			else {
+				sb.append_char(((0xF0 | (code >> 18)) as char))
+				sb.append_char(((0x80 | ((code >> 12) & 0x3F)) as char))
+				sb.append_char(((0x80 | ((code >> 6) & 0x3F)) as char))
+				sb.append_char(((0x80 | (code & 0x3F)) as char))
+			}
+		}
+		return sb.to_string()
+	}
+
+	// Count the code points in text (one pass; malformed bytes each count
+	// as one U+FFFD).
+	pub func char_count = (view string text, out int) {
+		var int n = 0
+		var int pos = 0
+		var int len = text.length
+		while pos < len {
+			pos = pos + Utf8.width_at(text, pos)
+			n = n + 1
+		}
+		return n
+	}
+}
+
+// A continuation byte in [lo, hi] at position i, or -1 when i is past the
+// end or the byte is not a continuation in range.
+func cont_byte = (view string text, int i, int lo, int hi, out int) {
+	if i < 0 || i >= text.length {
+		return -1
+	}
+	var int b = text.at(i) as int
+	if b < lo || b > hi {
+		return -1
+	}
+	return b
+}

package/core/System/char.nm

@@ -44,4 +44,29 @@ pub struct char: Stringable, Hashable, Equatable {
 		.Lend_char_to_string:
 		```
 	}
+
+	// ASCII digit: '0'-'9'. Non-ASCII bytes are false.
+	pub func is_digit = (self, out bool) {
+		var int code = self as int
+		return code >= 48 && code <= 57
+	}
+
+	// ASCII letter: 'A'-'Z' or 'a'-'z'. Non-ASCII bytes are false.
+	pub func is_alpha = (self, out bool) {
+		var int code = self as int
+		return (code >= 65 && code <= 90) || (code >= 97 && code <= 122)
+	}
+
+	// ASCII letter or digit.
+	pub func is_alphanumeric = (self, out bool) {
+		return self.is_alpha() || self.is_digit()
+	}
+
+	// ASCII whitespace: tab, LF, VT, FF, CR, space. Unicode spaces are
+	// false — byte-oriented callers trim with this and pass multibyte
+	// sequences through untouched.
+	pub func is_ascii_space = (self, out bool) {
+		var int code = self as int
+		return (code >= 9 && code <= 13) || code == 32
+	}
 }