mt-lang 0.2.0 → 0.2.1

data/std/str.mt

@@ -0,0 +1,293 @@
+import std.mem.arena as arena
+
+
+public function cstr_len(text: cstr) -> ptr_uint:
+    var count: ptr_uint = 0
+    unsafe:
+        let data = ptr[char]<-text
+        while read(data + count) != zero[char]:
+            count += 1
+    return count
+
+
+public function cstr_as_str(text: cstr) -> str:
+    return unsafe: str(data = ptr[char]<-text, len = cstr_len(text))
+
+
+public function chars_as_str(text: ptr[char]) -> str:
+    return unsafe: str(data = text, len = cstr_len(cstr<-text))
+
+
+public function nullable_cstr_as_str(text: cstr?) -> Option[str]:
+    if text == null:
+        return Option[str].none
+
+    return Option[str].some(value= cstr_as_str(text))
+
+
+public function as_byte_span(text: str) -> span[ubyte]:
+    return unsafe: span[ubyte](data = ptr[ubyte]<-text.data, len = text.len)
+
+
+public function utf8_byte_span_as_str(bytes: span[ubyte]) -> Option[str]:
+    unsafe:
+        let borrowed = str(data = ptr[char]<-bytes.data, len = bytes.len)
+        if not borrowed.is_valid_utf8():
+            return Option[str].none
+
+        return Option[str].some(value= borrowed)
+
+
+public function utf8_continuation_byte(value: ubyte) -> bool:
+    return (value & 0xC0) == 0x80
+
+
+function utf8_boundary(text: str, index: ptr_uint) -> bool:
+    if index == 0 or index == text.len:
+        return true
+
+    unsafe:
+        let value = ubyte<-read(text.data + index)
+        return not utf8_continuation_byte(value)
+
+
+function is_ascii_space(value: ubyte) -> bool:
+    return value == 32 or value == 9 or value == 10 or value == 13 or value == 12
+
+
+extending str:
+    public function byte_at(index: ptr_uint) -> ubyte:
+        if index >= this.len:
+            fatal(c"str.byte_at index out of bounds")
+
+        return unsafe: ubyte<-read(this.data + index)
+
+
+    public function equal(right: str) -> bool:
+        if this.len != right.len:
+            return false
+
+        var index: ptr_uint = 0
+        while index < this.len:
+            if this.byte_at(index) != right.byte_at(index):
+                return false
+            index += 1
+        return true
+
+
+    public function starts_with(prefix: str) -> bool:
+        if prefix.len > this.len:
+            return false
+
+        var index: ptr_uint = 0
+        while index < prefix.len:
+            if this.byte_at(index) != prefix.byte_at(index):
+                return false
+            index += 1
+        return true
+
+
+    public function ends_with(suffix: str) -> bool:
+        if suffix.len > this.len:
+            return false
+
+        let offset = this.len - suffix.len
+        var index: ptr_uint = 0
+        while index < suffix.len:
+            if this.byte_at(offset + index) != suffix.byte_at(index):
+                return false
+            index += 1
+        return true
+
+
+    public function find_byte(value: ubyte) -> Option[ptr_uint]:
+        var index: ptr_uint = 0
+        while index < this.len:
+            if this.byte_at(index) == value:
+                return Option[ptr_uint].some(value= index)
+            index += 1
+        return Option[ptr_uint].none
+
+
+    public function trim_ascii_whitespace() -> str:
+        var start: ptr_uint = 0
+        while start < this.len and is_ascii_space(this.byte_at(start)):
+            start += 1
+
+        var stop = this.len
+        while stop > start and is_ascii_space(this.byte_at(stop - 1)):
+            stop -= 1
+
+        return unsafe: str(data = this.data + start, len = stop - start)
+
+
+    public function is_valid_utf8() -> bool:
+        var index: ptr_uint = 0
+        while index < this.len:
+            let first = this.byte_at(index)
+            if first < 0x80:
+                index += 1
+            else if first >= 0xC2 and first <= 0xDF:
+                if index + 1 >= this.len or not utf8_continuation_byte(this.byte_at(index + 1)):
+                    return false
+                index += 2
+            else if first == 0xE0:
+                if index + 2 >= this.len:
+                    return false
+                let second = this.byte_at(index + 1)
+                if second < 0xA0 or second > 0xBF or not utf8_continuation_byte(this.byte_at(index + 2)):
+                    return false
+                index += 3
+            else if first >= 0xE1 and first <= 0xEC:
+                if (
+                    index + 2 >= this.len
+                    or not utf8_continuation_byte(this.byte_at(index + 1))
+                    or not utf8_continuation_byte(this.byte_at(index + 2))
+                ):
+                    return false
+                index += 3
+            else if first == 0xED:
+                if index + 2 >= this.len:
+                    return false
+                let second = this.byte_at(index + 1)
+                if second < 0x80 or second > 0x9F or not utf8_continuation_byte(this.byte_at(index + 2)):
+                    return false
+                index += 3
+            else if first >= 0xEE and first <= 0xEF:
+                if (
+                    index + 2 >= this.len
+                    or not utf8_continuation_byte(this.byte_at(index + 1))
+                    or not utf8_continuation_byte(this.byte_at(index + 2))
+                ):
+                    return false
+                index += 3
+            else if first == 0xF0:
+                if index + 3 >= this.len:
+                    return false
+                let second = this.byte_at(index + 1)
+                if (
+                    second < 0x90
+                    or second > 0xBF
+                    or not utf8_continuation_byte(this.byte_at(index + 2))
+                    or not utf8_continuation_byte(this.byte_at(index + 3))
+                ):
+                    return false
+                index += 4
+            else if first >= 0xF1 and first <= 0xF3:
+                if (
+                    index + 3 >= this.len
+                    or not utf8_continuation_byte(this.byte_at(index + 1))
+                    or not utf8_continuation_byte(this.byte_at(index + 2))
+                    or not utf8_continuation_byte(this.byte_at(index + 3))
+                ):
+                    return false
+                index += 4
+            else if first == 0xF4:
+                if index + 3 >= this.len:
+                    return false
+                let second = this.byte_at(index + 1)
+                if (
+                    second < 0x80
+                    or second > 0x8F
+                    or not utf8_continuation_byte(this.byte_at(index + 2))
+                    or not utf8_continuation_byte(this.byte_at(index + 3))
+                ):
+                    return false
+                index += 4
+            else:
+                return false
+
+        return true
+
+
+    public function slice(start: ptr_uint, len: ptr_uint) -> str:
+        if start > this.len:
+            fatal(c"str slice start out of bounds")
+        if len > this.len - start:
+            fatal(c"str slice length out of bounds")
+
+        if not utf8_boundary(this, start):
+            fatal(c"str slice start must be a UTF-8 boundary")
+        var actual_stop = start + len
+        if not utf8_boundary(this, actual_stop):
+            fatal(c"str slice end must be a UTF-8 boundary")
+
+        return unsafe: str(data = this.data + start, len = actual_stop - start)
+
+
+    public function find_substring(needle: str) -> Option[ptr_uint]:
+        if needle.len == 0:
+            return Option[ptr_uint].some(value= 0)
+        if needle.len > this.len:
+            return Option[ptr_uint].none
+
+        let limit = this.len - needle.len
+        var offset: ptr_uint = 0
+        while offset <= limit:
+            var matched = true
+            var index: ptr_uint = 0
+            while index < needle.len:
+                if this.byte_at(offset + index) != needle.byte_at(index):
+                    matched = false
+                    break
+                index += 1
+            if matched:
+                return Option[ptr_uint].some(value= offset)
+            offset += 1
+
+        return Option[ptr_uint].none
+
+
+    public function contains_substring(needle: str) -> bool:
+        return this.find_substring(needle).is_some()
+
+
+    public static function hash(value: const_ptr[str]) -> uint:
+        let fnv_offset: uint = 0x811C9DC5
+        let fnv_prime: uint = 0x01000193
+
+        var result = fnv_offset
+        unsafe:
+            let view = read(value)
+            var index: ptr_uint = 0
+            while index < view.len:
+                let byte_value = uint<-view.byte_at(index)
+                result = (result ^ byte_value) * fnv_prime
+                index += 1
+        return result
+
+
+    public static function equal(left: const_ptr[str], right: const_ptr[str]) -> bool:
+        unsafe:
+            return read(left).equal(read(right))
+
+
+    public static function order(left: const_ptr[str], right: const_ptr[str]) -> int:
+        unsafe:
+            let left_view = read(left)
+            let right_view = read(right)
+            var min_len = left_view.len
+            if right_view.len < min_len:
+                min_len = right_view.len
+
+            var index: ptr_uint = 0
+            while index < min_len:
+                let lb = int<-left_view.byte_at(index)
+                let rb = int<-right_view.byte_at(index)
+                if lb != rb:
+                    return lb - rb
+                index += 1
+
+            let left_len = int<-left_view.len
+            let right_len = int<-right_view.len
+            return left_len - right_len
+
+
+    public function compare(right: str) -> int:
+        var left_copy: str = this
+        var right_copy: str = right
+        return str.order(ptr_of(left_copy), ptr_of(right_copy))
+
+
+    public function to_cstr(space: ref[arena.Arena]) -> cstr:
+        return space.to_cstr(this)

data/std/string.mt

@@ -0,0 +1,234 @@
+import std.str as text_ops
+import std.mem.arena as arena
+import std.mem.heap as heap
+import std.vec as vec
+
+public struct String:
+    data: ptr[ubyte]?
+    len: ptr_uint
+    capacity: ptr_uint
+    owns_storage: bool
+
+
+extending String:
+    public static function create() -> String:
+        return String(data = null, len = 0, capacity = 0, owns_storage = true)
+
+
+    public static function with_capacity(capacity: ptr_uint) -> String:
+        var result = String.create()
+        result.reserve(capacity)
+        return result
+
+
+    public static function from_str(text: str) -> String:
+        var result = String.with_capacity(text.len)
+        result.append(text)
+        return result
+
+
+    public function len() -> ptr_uint:
+        return this.len
+
+
+    public function capacity() -> ptr_uint:
+        return this.capacity
+
+
+    public function is_empty() -> bool:
+        return this.len == 0
+
+
+    public function equal(other: String) -> bool:
+        return this.as_str().equal(other.as_str())
+
+
+    public editable function clear() -> void:
+        this.len = 0
+
+
+    public editable function truncate(new_len: ptr_uint) -> void:
+        if new_len > this.len:
+            fatal(c"string.truncate new length exceeds current length")
+        this.len = new_len
+
+
+    public editable function release() -> void:
+        if this.owns_storage:
+            heap.release(this.data)
+
+        this.data = null
+        this.len = 0
+        this.capacity = 0
+        this.owns_storage = true
+
+
+    public static function hash(value: const_ptr[String]) -> uint:
+        let fnv_offset: uint = 0x811C9DC5
+        let fnv_prime: uint = 0x01000193
+
+        unsafe:
+            let view = read(value)
+            var result = fnv_offset
+            let raw = view.as_str()
+            var index: ptr_uint = 0
+            while index < raw.len:
+                let byte_value = uint<-raw.byte_at(index)
+                result = (result ^ byte_value) * fnv_prime
+                index += 1
+            return result
+
+
+    public static function equal(left: const_ptr[String], right: const_ptr[String]) -> bool:
+        unsafe:
+            let left_view = read(left)
+            let right_view = read(right)
+            return left_view.as_str().equal(right_view.as_str())
+
+
+    public static function order(left: const_ptr[String], right: const_ptr[String]) -> int:
+        unsafe:
+            let left_view = read(left)
+            let right_view = read(right)
+            return left_view.as_str().compare(right_view.as_str())
+
+
+    public function starts_with(prefix: str) -> bool:
+        return this.as_str().starts_with(prefix)
+
+
+    public function ends_with(suffix: str) -> bool:
+        return this.as_str().ends_with(suffix)
+
+
+    public function find_substring(needle: str) -> Option[ptr_uint]:
+        return this.as_str().find_substring(needle)
+
+
+    public function contains_substring(needle: str) -> bool:
+        return this.as_str().contains_substring(needle)
+
+
+    public editable function reserve(min_capacity: ptr_uint) -> void:
+        if min_capacity <= this.capacity:
+            return
+
+        if not this.owns_storage:
+            fatal(c"string.reserve cannot grow borrowed storage")
+
+        var new_capacity = this.capacity
+        if new_capacity == 0:
+            new_capacity = 4
+
+        while new_capacity < min_capacity:
+            if new_capacity > heap.ptr_uint_max / 2:
+                new_capacity = min_capacity
+            else:
+                new_capacity *= 2
+
+        let resized = heap.resize[ubyte](this.data, new_capacity) else:
+            fatal(c"string.reserve out of memory")
+
+        this.data = resized
+        this.capacity = new_capacity
+
+
+    public editable function push_byte(value: ubyte) -> void:
+        if this.len == this.capacity:
+            this.reserve(this.len + 1)
+
+        let data = this.data else:
+            fatal(c"string.push_byte missing storage")
+
+        unsafe:
+            let data_ptr = data
+            read(data_ptr + this.len) = value
+        this.len += 1
+
+
+    public editable function append(suffix: str) -> void:
+        if suffix.len == 0:
+            return
+
+        let current_len = this.len
+        if suffix.len > heap.ptr_uint_max - current_len:
+            fatal(c"string.append size overflow")
+
+        let new_len = current_len + suffix.len
+        let needs_growth = new_len > this.capacity
+
+        var copied: ptr[ubyte]? = null
+        if needs_growth:
+            copied = heap.must_alloc[ubyte](suffix.len)
+            unsafe: heap.copy_bytes(copied, ptr[ubyte]<-suffix.data, suffix.len)
+
+        this.reserve(new_len)
+
+        let data = this.data else:
+            fatal(c"string.append missing storage")
+
+        unsafe:
+            let destination = data + current_len
+            if copied != null:
+                heap.copy_bytes(destination, copied, suffix.len)
+                heap.release(copied)
+            else:
+                heap.copy_bytes(destination, ptr[ubyte]<-suffix.data, suffix.len)
+
+        this.len = new_len
+
+
+    public editable function append_format(text: str) -> void:
+        this.append(text)
+
+
+    public editable function assign(value_text: str) -> void:
+        this.clear()
+        this.append(value_text)
+
+
+    public editable function assign_format(text: str) -> void:
+        this.assign(text)
+
+
+    public function as_str() -> str:
+        let data = this.data
+        if data == null and this.len != 0:
+            fatal(c"string.as_str requires storage when len > 0")
+
+        let borrowed = unsafe: str(data = ptr[char]<-data, len = this.len)
+        if not borrowed.is_valid_utf8():
+            fatal(c"string.as_str text must be valid UTF-8")
+
+        return borrowed
+
+
+    public function to_cstr(space: ref[arena.Arena]) -> cstr:
+        return space.to_cstr(this.as_str())
+
+
+    public function split(separator: str) -> vec.Vec[String]:
+        var result = vec.Vec[String].create()
+        var remaining = this.as_str()
+        while true:
+            let found = remaining.find_substring(separator)
+            let idx = found else:
+                break
+            result.push(String.from_str(remaining.slice(0, idx)))
+            remaining = remaining.slice(idx + separator.len, remaining.len - idx - separator.len)
+        result.push(String.from_str(remaining))
+        return result
+
+
+    public function replace(old_substr: str, new_substr: str) -> String:
+        var result = String.create()
+        var remaining = this.as_str()
+        while true:
+            let found = remaining.find_substring(old_substr)
+            let idx = found else:
+                break
+            result.append(remaining.slice(0, idx))
+            result.append(new_substr)
+            remaining = remaining.slice(idx + old_substr.len, remaining.len - idx - old_substr.len)
+        result.append(remaining)
+        return result