nomen-lang 0.2.3 → 0.3.0

package/core/System/String.nm

@@ -3,6 +3,19 @@
 // result is re-wrapped with its length.
 extern func strdup = (string s, move out string)
 
+// Raw memory externs — the allocation primitives every `unsafe` Nomen body
+// across core (Buffer, StringBuilder, Array, ...) is built on. This file is
+// part of the always-linked base types, so the declarations are visible to
+// every core file regardless of pull order. Free externs emit under an
+// `extern_<name>` label that wraps the identically-named C symbol, so these
+// can never collide with it (docs/CORE_RAW.md roadmap items 1+2).
+extern func calloc = (int count, int size, out uint64)
+extern func malloc = (int bytes, out uint64)
+extern func realloc = (uint64 address, int bytes, out uint64)
+extern func free = (uint64 address)
+extern func memcpy = (uint64 dst, uint64 src, int bytes, out uint64)
+extern func memset = (uint64 dst, int value, int bytes, out uint64)
+
 /**
  * A UTF-8 text string — a heap-owned, length-tracked sequence of bytes
  **/
@@ -28,17 +41,13 @@ pub struct string: Stringable, Hashable, Equatable {
 		return strdup(self)
 	}
 
+	// The indexing primitive everything string code builds on — `unsafe`
+	// Nomen, single-sourced for both backends. `self as ptr char` is the fat
+	// value's backing bytes (the thin `char*` the raw blocks used to poke).
 	pub func at = (self, int index: index >= 0 && index < self.length, out char) {
-		```
-		#arch: c
-		return self[index];
-		```
-		```
-		#arch: aarch64
-		// Fat-string ABI: x19 = self ptr, x20 = self len; index in x2
-		// (self's pair occupies slots 0-1).
-		ldrb w0, [x19, x2]
-		```
+		unsafe {
+			return (self as ptr char)[index]
+		}
 	}
 
 	// Bounds-checked access: `fallback` when index is outside
@@ -87,15 +96,9 @@ pub struct string: Stringable, Hashable, Equatable {
 	}
 
 	pub func set = (ref self, int index: index >= 0 && index < self.length, char value) {
-		```
-		#arch: c
-		(*self)[index] = value;
-		```
-		```
-		#arch: aarch64
-		// x19 = self (pointer), x1 = index, w2 = value
-		strb w2, [x19, x1]
-		```
+		unsafe {
+			(self as ptr char)[index] = value
+		}
 	}
 
 	pub func #op_eq = (self, string other, out bool) {
@@ -209,4 +212,238 @@ pub struct string: Stringable, Hashable, Equatable {
 		ldp x20, x21, [sp], #16
 		```
 	}
+
+	// Replace the first non-overlapping occurrence of needle with
+	// replacement. No match — or an empty needle — returns an owned copy
+	// of self. Views pass straight in (no `to_string` copy): both sides
+	// are only ever read.
+	pub func replace_first = (self, view string needle, view string replacement, out string) {
+		var sb = StringBuilder()
+		var int i = 0
+		var int n = self.length
+		var int nlen = needle.length
+		var bool replaced = nlen == 0
+		while i < n {
+			if !replaced && nlen > 0 && i + nlen <= n && match_at(self, needle, i) {
+				sb.append_string_view(replacement)
+				i = i + nlen
+				replaced = true
+			} else {
+				sb.append_char(self.at(i))
+				i = i + 1
+			}
+		}
+		return sb.to_string()
+	}
+
+	// Replace all non-overlapping occurrences of needle with replacement,
+	// left to right. Same edge behavior as replace_first.
+	pub func replace_all = (self, view string needle, view string replacement, out string) {
+		var sb = StringBuilder()
+		var int i = 0
+		var int n = self.length
+		var int nlen = needle.length
+		while i < n {
+			if nlen > 0 && i + nlen <= n && match_at(self, needle, i) {
+				sb.append_string_view(replacement)
+				i = i + nlen
+			} else {
+				sb.append_char(self.at(i))
+				i = i + 1
+			}
+		}
+		return sb.to_string()
+	}
+
+	// First index of `needle` in self, or -1 when absent. An empty needle
+	// matches at 0 (JavaScript indexOf semantics).
+	pub func index_of = (self, view string needle, out int) {
+		return self.index_of_from(needle, 0)
+	}
+
+	// First index of `needle` at or after byte offset `from`, or -1. `from`
+	// is clamped into [0, self.length]; an empty needle returns the clamped
+	// `from`, so a `from` past the end yields self.length for the empty
+	// needle and -1 otherwise.
+	pub func index_of_from = (self, view string needle, int from, out int) {
+		var int f = from
+		if f < 0 {
+			f = 0
+		}
+		if f > self.length {
+			f = self.length
+		}
+		if needle.length == 0 {
+			return f
+		}
+		var int last = self.length - needle.length
+		var int i = f
+		while i <= last; i += 1 {
+			if match_at(self, needle, i) {
+				return i
+			}
+		}
+		return -1
+	}
+
+	// Whether `needle` occurs anywhere in self.
+	pub func contains = (self, view string needle, out bool) {
+		return self.index_of(needle) >= 0
+	}
+
+	// Whether self begins with `prefix`.
+	pub func starts_with = (self, view string prefix, out bool) {
+		if prefix.length > self.length {
+			return false
+		}
+		return match_at(self, prefix, 0)
+	}
+
+	// Whether self ends with `suffix`.
+	pub func ends_with = (self, view string suffix, out bool) {
+		if suffix.length > self.length {
+			return false
+		}
+		return match_at(self, suffix, self.length - suffix.length)
+	}
+
+	// The code of the byte at `index`, or -1 when out of bounds — the
+	// JavaScript charCodeAt NaN case as a plain int parsers can test.
+	// The code of the byte at `index`. Like `at`, the constraint makes
+	// out-of-bounds reads unrepresentable: a literal index is verified at
+	// compile time, and a dynamic index must be proven in-range by flow
+	// facts (a `while i < self.length` loop or an `if` guard) — an
+	// unverifiable call is a compile error. For a sentinel read, use
+	// `char_code_at_or`; for a trapping read, `char_code_at_or_panic`.
+	pub func char_code_at = (self, int index: index >= 0 && index < self.length, out int) {
+		return self.at(index) as int
+	}
+
+	// The code of the byte at `index`, with `fallback` when index is
+	// outside [0, self.length).
+	pub func char_code_at_or = (self, int index, int fallback, out int) {
+		if index >= 0 && index < self.length {
+			return self.at(index) as int
+		}
+		return fallback
+	}
+
+	// The code of the byte at `index`; traps when index is outside
+	// [0, self.length) — out-of-range is a bug, not a case.
+	pub func char_code_at_or_panic = (self, int index, out int) {
+		if index >= 0 && index < self.length {
+			return self.at(index) as int
+		}
+		panic("index out of range")
+	}
+
+	// Owned copy of self[start, end) with both bounds clamped into
+	// [0, self.length] and an inverted pair swapped (JavaScript substring
+	// semantics: out-of-range operands can never trap).
+	pub func substring = (self, int start, int end, out string) {
+		var int a = start
+		var int b = end
+		if a < 0 {
+			a = 0
+		}
+		if a > self.length {
+			a = self.length
+		}
+		if b > self.length {
+			b = self.length
+		}
+		if b < a {
+			b = a
+		}
+		var view v = self.slice(a, b)
+		return v.to_string()
+	}
+
+	// Owned copy of self with leading and trailing ASCII whitespace
+	// (tab, LF, VT, FF, CR, space) removed. All-whitespace yields "".
+	pub func trim = (self, out string) {
+		var int start = 0
+		while start < self.length; start += 1 {
+			if !self.at(start).is_ascii_space() {
+				break
+			}
+		}
+		var int end = self.length
+		while end > start; end -= 1 {
+			if !self.at(end - 1).is_ascii_space() {
+				break
+			}
+		}
+		var view v = self.slice(start, end)
+		return v.to_string()
+	}
+
+	// Owned copy of self with trailing ASCII whitespace removed.
+	pub func trim_end = (self, out string) {
+		var int end = self.length
+		while end > 0; end -= 1 {
+			if !self.at(end - 1).is_ascii_space() {
+				break
+			}
+		}
+		var view v = self.slice(0, end)
+		return v.to_string()
+	}
+
+	// Owned copy of self with leading ASCII whitespace removed.
+	pub func trim_start = (self, out string) {
+		var int start = 0
+		while start < self.length; start += 1 {
+			if !self.at(start).is_ascii_space() {
+				break
+			}
+		}
+		var view v = self.slice(start, self.length)
+		return v.to_string()
+	}
+
+	// Owned ASCII-lowercased copy of self (A-Z mapped to a-z; all other
+	// bytes — including UTF-8 sequences — pass through unchanged).
+	pub func to_lowercase = (self, out string) {
+		var sb = StringBuilder()
+		var int i = 0
+		while i < self.length; i += 1 {
+			var int code = self.at(i) as int
+			if code > 64 && code < 91 {
+				code += 32
+			}
+			sb.append_char(code as char)
+		}
+		return sb.to_string()
+	}
+
+	// Owned ASCII-uppercased copy of self (a-z mapped to A-Z; all other
+	// bytes — including UTF-8 sequences — pass through unchanged).
+	pub func to_uppercase = (self, out string) {
+		var sb = StringBuilder()
+		var int i = 0
+		while i < self.length; i += 1 {
+			var int code = self.at(i) as int
+			if code > 96 && code < 123 {
+				code -= 32
+			}
+			sb.append_char(code as char)
+		}
+		return sb.to_string()
+	}
+}
+
+// needle matches text at byte offset pos (caller guarantees pos + len fits;
+// the trailing guard makes the bound explicit so `.at` verifies).
+func match_at = (view string text, view string needle, int pos, out bool) {
+	var int i = 0
+	var int nlen = needle.length
+	var int n = text.length
+	while i < nlen && pos + i < n {
+		if text.at(pos + i) != needle.at(i) {
+			return false
+		}
+		i = i + 1
+	}
+	return i == nlen
 }

package/core/System/StringBuilder.nm

@@ -14,48 +14,19 @@ pub struct StringBuilder {
 	var cap = 0
 
 	// Ensure capacity for at least `needed` bytes. Grows geometrically.
+	// Plain Nomen + the memory externs — no pointer arithmetic needed, so
+	// this doesn't even require an unsafe context.
 	func ensure = (ref self, int needed) {
-		```
-		#arch: c
-		if (self->cap >= needed) return;
-		int new_cap = self->cap;
-		if (new_cap == 0) new_cap = 16;
-		while (new_cap < needed) new_cap *= 2;
-		self->data = (unsigned long long)realloc((void*)(unsigned long long)self->data, new_cap);
-		self->cap = new_cap;
-		```
-		```
-		#arch: aarch64
-		// x19 = self, x1 = needed
-		ldr x2, [x19, #24]        // cap
-		cmp x2, x1
-		b.ge .Lsb_ensure_end
-		// pick new_cap
-		mov x3, #16
-		cmp x2, #0
-		csel x3, x3, x2, eq
-		.Lsb_ensure_loop:
-		cmp x3, x1
-		b.ge .Lsb_ensure_grow
-		lsl x3, x3, #1
-		b .Lsb_ensure_loop
-		.Lsb_ensure_grow:
-		// save needed (x1) and new_cap (x3) across realloc
-		sub sp, sp, #32
-		str x1, [sp, #0]
-		str x3, [sp, #8]
-		str x2, [sp, #16]         // old cap (unused after, but keeps stack aligned)
-		ldr x0, [x19, #8]         // data
-		mov x1, x3
-		bl _realloc
-		str x0, [sp, #24]         // saved new data ptr
-		ldr x1, [sp, #8]          // new_cap
-		str x1, [x19, #24]        // cap = new_cap
-		ldr x0, [sp, #24]
-		str x0, [x19, #8]         // data = realloc result
-		add sp, sp, #32
-		.Lsb_ensure_end:
-		```
+		if self.cap >= needed {
+			return
+		}
+		var int new_cap = self.cap
+		if new_cap == 0 {
+			new_cap = 16
+		}
+		while new_cap < needed; new_cap *= 2 {}
+		self.data = realloc(self.data, new_cap)
+		self.cap = new_cap
 	}
 
 	// Append a single byte (char).
@@ -110,65 +81,43 @@ pub struct StringBuilder {
 		```
 	}
 
-	// Append a null-terminated string.
+	// Append a null-terminated string. `unsafe` Nomen single-sourced for
+	// both backends: the fat pair's pointer half feeds memcpy directly, and
+	// the tracked length replaces the strlen the old C body paid.
 	func append_string = (ref self, string s) {
-		```
-		#arch: c
-		int sl = strlen(s);
-		int needed = self->len + sl + 1;
-		if (needed > self->cap) {
-			int nc = self->cap;
-			if (nc == 0) nc = 16;
-			while (nc < needed) nc *= 2;
-			self->data = (unsigned long long)realloc((void*)(unsigned long long)self->data, nc);
-			self->cap = nc;
+		unsafe {
+			var int sl = s.length
+			self.ensure(self.len + sl + 1)
+			memcpy(self.data + self.len, (s as ptr char) as uint64, sl)
+			self.len = self.len + sl
+		}
+	}
+
+	// Append a view's bytes without materializing it: `append_string` takes
+	// an owned string, so passing a slice would need a `to_string` copy per
+	// call. Views lower to the same (ptr, len) fat pair, so the bytes are
+	// memcpy'd straight out of the borrowed buffer (used by
+	// `string.replace_first`/`replace_all`).
+	pub func append_string_view = (ref self, view string s) {
+		unsafe {
+			var int sl = s.length
+			self.ensure(self.len + sl + 1)
+			memcpy(self.data + self.len, (s as ptr char) as uint64, sl)
+			self.len = self.len + sl
+		}
+	}
+
+	// Replace the buffer's contents with a copy of `s`'s bytes. The only
+	// way to start a StringBuilder from a slice without a `to_string()`
+	// copy: the view's bytes are copied once into fresh storage, so the
+	// result is owned and independent of the source.
+	pub func seed = (ref self, view string s) {
+		unsafe {
+			var int sl = s.length
+			self.ensure(sl + 1)
+			memcpy(self.data, (s as ptr char) as uint64, sl)
+			self.len = sl
 		}
-		memcpy((unsigned char*)(unsigned long long)self->data + self->len, s, sl);
-		self->len = self->len + sl;
-		```
-		```
-		#arch: aarch64
-		// Fat-string ABI: x19 = self; s = (x1 ptr, x2 len). The length is
-		// already in a register — no strlen.
-		sub sp, sp, #32
-		str x1, [sp, #0]          // s ptr
-		str x2, [sp, #8]          // sl
-		ldr x1, [x19, #16]        // len
-		add x1, x1, x2            // len + sl
-		add x1, x1, #1            // +1 for null room
-		// inline ensure(x1)
-		ldr x2, [x19, #24]
-		cmp x2, x1
-		b.ge .Lsb_as_copy
-		mov x3, #16
-		cmp x2, #0
-		csel x3, x3, x2, eq
-		.Lsb_as_loop:
-		cmp x3, x1
-		b.ge .Lsb_as_grow
-		lsl x3, x3, #1
-		b .Lsb_as_loop
-		.Lsb_as_grow:
-		str x3, [x19, #24]
-		ldr x0, [x19, #8]
-		mov x1, x3
-		bl _realloc
-		str x0, [x19, #8]
-		.Lsb_as_copy:
-		// memcpy(data + len, s.ptr, sl)
-		ldr x0, [x19, #8]         // data
-		ldr x1, [x19, #16]        // len
-		add x0, x0, x1            // dest = data + len
-		ldr x1, [sp, #0]          // src = s ptr
-		ldr x2, [sp, #8]          // n = sl
-		bl _memcpy
-		// len = len + sl
-		ldr x0, [x19, #16]
-		ldr x1, [sp, #8]
-		add x0, x0, x1
-		str x0, [x19, #16]
-		add sp, sp, #32
-		```
 	}
 
 	// Finalize: null-terminate, return the byte buffer as a string, and
@@ -188,7 +137,7 @@ pub struct StringBuilder {
 		((unsigned char*)(unsigned long long)self->data)[idx] = 0;
 		unsigned long long result = self->data;
 		self->data = 0;
-		return (char*)result;
+		return (nomen_string){ (char*)result, (long)idx };
 		```
 		```
 		#arch: aarch64
@@ -224,24 +173,11 @@ pub struct StringBuilder {
 
 	// Drop the buffer without finalizing (free the byte buffer outright).
 	func #destroy = () {
-		```
-		#arch: c
-		if (self->data) {
-			free((void*)(unsigned long long)self->data);
-			self->data = 0;
-			self->len = 0;
-			self->cap = 0;
+		if self.data != 0 {
+			free(self.data)
+			self.data = 0
+			self.len = 0
+			self.cap = 0
 		}
-		```
-		```
-		#arch: aarch64
-		ldr x0, [x19, #8]
-		cbz x0, .Lsb_destroy_end
-		bl _free
-		str xzr, [x19, #8]
-		str xzr, [x19, #16]
-		str xzr, [x19, #24]
-		.Lsb_destroy_end:
-		```
 	}
 }

package/core/System/Text/CharIndex.nm

@@ -0,0 +1,144 @@
+// A run-length index of character widths for O(log n) random access.
+//
+// ASCII text needs no records at all (width-1 runs are implicit); only runs
+// of multi-byte characters are stored, each as {byte_index, char_start,
+// width, count}. All lookups binary-search the runs, so repeated indexed
+// access stays logarithmic no matter how often callers re-walk. The index
+// borrows its source (no copy of the text).
+
+/**
+ * A run of consecutive same-width multi-byte characters
+ **/
+pub struct CharRun {
+	var int byte_index
+	var int char_start
+	var int width
+	var int count
+}
+
+/**
+ * A run-length character-width index (construct with `CharIndex(text)`,
+ * then `byte_offset_of` / `char_index_of` / `char_at` in O(log runs))
+ **/
+pub struct CharIndex {
+	var view string source
+	var List<CharRun> runs = List<CharRun>()
+	var int char_count = 0
+
+	pub func #init = (self, view string source) {
+		self.source = source
+		var int pos = 0
+		var int len = source.length
+		var int chars = 0
+		var int open_at = -1
+		var int open_width = 0
+		var int open_start = 0
+		var int open_count = 0
+		while pos < len {
+			var int w = Utf8.width_at(source, pos)
+			switch {
+				case w == 1 {
+					if open_at >= 0 {
+						self.runs.push(CharRun(open_at, open_start, open_width, open_count))
+						open_at = -1
+					}
+					pos = pos + 1
+					chars = chars + 1
+				}
+				case open_at >= 0 && w == open_width {
+					open_count = open_count + 1
+					pos = pos + w
+					chars = chars + 1
+				}
+				else {
+					if open_at >= 0 {
+						self.runs.push(CharRun(open_at, open_start, open_width, open_count))
+					}
+					open_at = pos
+					open_start = chars
+					open_width = w
+					open_count = 1
+					pos = pos + w
+					chars = chars + 1
+				}
+			}
+		}
+		if open_at >= 0 {
+			self.runs.push(CharRun(open_at, open_start, open_width, open_count))
+		}
+		self.char_count = chars
+	}
+
+	// Index built over source in one pass. Construct directly —
+	// `CharIndex(text)` borrows the caller's text with no copy.
+	pub func byte_offset_of = (self, int char_index, out int) {
+		if char_index < 0 || char_index >= self.char_count {
+			panic("character index out of range")
+		}
+		var int found = self.rightmost_run(char_index, true)
+		if found < 0 {
+			// No run starts at or before it: pure ASCII prefix.
+			return char_index
+		}
+		var r = self.runs.at_or_panic(found)
+		if char_index < r.char_start + r.count {
+			return r.byte_index + (char_index - r.char_start) * r.width
+		}
+		var int after_chars = r.char_start + r.count
+		var int after_bytes = r.byte_index + r.count * r.width
+		return after_bytes + (char_index - after_chars)
+	}
+
+	// Character index containing byte_offset. A mid-character offset floors
+	// into its character; byte_offset == source.length maps to char_count.
+	// Panics if byte_offset is out of [0, source.length].
+	pub func char_index_of = (self, int byte_offset, out int) {
+		if byte_offset < 0 || byte_offset > self.source.length {
+			panic("byte offset out of range")
+		}
+		if byte_offset == self.source.length {
+			return self.char_count
+		}
+		var int found = self.rightmost_run(byte_offset, false)
+		if found < 0 {
+			return byte_offset
+		}
+		var r = self.runs.at_or_panic(found)
+		if byte_offset < r.byte_index + r.count * r.width {
+			return r.char_start + (byte_offset - r.byte_index) / r.width
+		}
+		var int after_chars = r.char_start + r.count
+		var int after_bytes = r.byte_index + r.count * r.width
+		return after_chars + (byte_offset - after_bytes)
+	}
+
+	// Decoded code point of the char_index-th character. Panics if
+	// char_index is out of [0, char_count).
+	pub func char_at = (self, int char_index, out int) {
+		var int at = self.byte_offset_of(char_index)
+		var d = Utf8.decode_at(self.source, at)
+		return d.code_point
+	}
+	// Rightmost run whose key (char_start when by_char, else byte_index) is
+	// <= target, or -1 when all runs start after it.
+	func rightmost_run = (self, int target, bool by_char, out int) {
+		var int lo = 0
+		var int hi = self.runs.length - 1
+		var int found = -1
+		while lo <= hi {
+			var int mid = (lo + hi) / 2
+			var r = self.runs.at_or_panic(mid)
+			var int key = r.char_start
+			if !by_char {
+				key = r.byte_index
+			}
+			if key <= target {
+				found = mid
+				lo = mid + 1
+			} else {
+				hi = mid - 1
+			}
+		}
+		return found
+	}
+}

package/core/System/Text/Chars.nm

@@ -0,0 +1,32 @@
+// A single-pass cursor over a string's Unicode scalar values.
+//
+// The cursor borrows its source (no copy) and decodes lazily: each `next`
+// costs one decode step. For repeated random access, build a CharIndex
+// instead — re-walking from scratch on every access is O(n) per lookup.
+
+/**
+ * A single-pass cursor over Unicode scalar values (`has_next`/`next`)
+ **/
+pub struct Chars {
+	var view string source
+	var int byte_pos = 0
+
+	pub func #init = (self, view string source) {
+		self.source = source
+	}
+
+	pub func has_next = (self, out bool) {
+		return self.byte_pos < self.source.length
+	}
+
+	// The code point at the cursor, advancing past it. Malformed sequences
+	// yield U+FFFD. Panics when exhausted — check has_next first.
+	pub func next = (ref self, out int) {
+		if self.byte_pos >= self.source.length {
+			panic("Chars exhausted; check has_next first")
+		}
+		var d = Utf8.decode_at(self.source, self.byte_pos)
+		self.byte_pos = self.byte_pos + d.byte_length
+		return d.code_point
+	}
+}