@shd101wyy/yo 0.1.4 → 0.1.6

package/std/string/unicode.yo

@@ -0,0 +1,242 @@
+// Unicode-aware case conversion
+//
+// Provides Unicode-aware lowercase/uppercase conversion using C's towlower/towupper,
+// plus hand-coded tables for special case folding entries where one codepoint
+// maps to multiple codepoints.
+//
+// Example:
+//   { unicode_to_lowercase, unicode_to_uppercase } :: import "std/string/unicode";
+//
+//   lower := unicode_to_lowercase(`HELLO WÖRLD`);  // "hello wörld"
+//   upper := unicode_to_uppercase(`hello wörld`);  // "HELLO WÖRLD"
+
+open import "../string";
+{ ArrayList } :: import "../collections/array_list";
+
+// Declare towlower/towupper directly with i32 to avoid wint_t cast issues.
+// On all major platforms, wint_t is compatible with int/i32.
+c_include "<wctype.h>",
+  towlower :
+    fn(wc : i32) -> i32,
+  towupper :
+    fn(wc : i32) -> i32
+;
+
+// Result of decoding a single UTF-8 codepoint
+_DecodeResult :: struct(
+  codepoint : i32,
+  bytes_consumed : usize
+);
+
+// Decode a single UTF-8 codepoint from bytes at position i.
+_decode_utf8 :: (fn(bytes: ArrayList(u8), i: usize) -> _DecodeResult)({
+  (b0 : i32) = i32(bytes.get(i).unwrap());
+  cond(
+    // 1-byte ASCII
+    ((b0 & i32(0x80)) == i32(0)) => {
+      _DecodeResult(codepoint: b0, bytes_consumed: usize(1))
+    },
+    // 2-byte
+    ((b0 & i32(0xE0)) == i32(0xC0)) => {
+      (b1 : i32) = i32(bytes.get((i + usize(1))).unwrap());
+      (cp : i32) = (((b0 & i32(0x1F)) << i32(6)) | (b1 & i32(0x3F)));
+      _DecodeResult(codepoint: cp, bytes_consumed: usize(2))
+    },
+    // 3-byte
+    ((b0 & i32(0xF0)) == i32(0xE0)) => {
+      (b1 : i32) = i32(bytes.get((i + usize(1))).unwrap());
+      (b2 : i32) = i32(bytes.get((i + usize(2))).unwrap());
+      (cp : i32) = ((((b0 & i32(0x0F)) << i32(12)) | ((b1 & i32(0x3F)) << i32(6))) | (b2 & i32(0x3F)));
+      _DecodeResult(codepoint: cp, bytes_consumed: usize(3))
+    },
+    // 4-byte
+    true => {
+      (b1 : i32) = i32(bytes.get((i + usize(1))).unwrap());
+      (b2 : i32) = i32(bytes.get((i + usize(2))).unwrap());
+      (b3 : i32) = i32(bytes.get((i + usize(3))).unwrap());
+      (cp : i32) = (((((b0 & i32(0x07)) << i32(18)) | ((b1 & i32(0x3F)) << i32(12))) | ((b2 & i32(0x3F)) << i32(6))) | (b3 & i32(0x3F)));
+      _DecodeResult(codepoint: cp, bytes_consumed: usize(4))
+    }
+  )
+});
+
+// Encode a Unicode codepoint as UTF-8 bytes into an ArrayList.
+_encode_utf8 :: (fn(cp: i32, out: *(ArrayList(u8))) -> unit)({
+  cond(
+    (cp < i32(0x80)) => {
+      out.*.push(u8(cp));
+    },
+    (cp < i32(0x800)) => {
+      out.*.push(u8((i32(0xC0) | (cp >> i32(6)))));
+      out.*.push(u8((i32(0x80) | (cp & i32(0x3F)))));
+    },
+    (cp < i32(0x10000)) => {
+      out.*.push(u8((i32(0xE0) | (cp >> i32(12)))));
+      out.*.push(u8((i32(0x80) | ((cp >> i32(6)) & i32(0x3F)))));
+      out.*.push(u8((i32(0x80) | (cp & i32(0x3F)))));
+    },
+    true => {
+      out.*.push(u8((i32(0xF0) | (cp >> i32(18)))));
+      out.*.push(u8((i32(0x80) | ((cp >> i32(12)) & i32(0x3F)))));
+      out.*.push(u8((i32(0x80) | ((cp >> i32(6)) & i32(0x3F)))));
+      out.*.push(u8((i32(0x80) | (cp & i32(0x3F)))));
+    }
+  );
+});
+
+// Special case folding: codepoints that expand to multiple codepoints
+// when lowercased. These are Unicode case folding entries of type 'F' (full).
+_special_to_lower :: (fn(cp: i32, out: *(ArrayList(u8))) -> bool)({
+  cond(
+    // ẞ (U+1E9E LATIN CAPITAL LETTER SHARP S) → ss
+    (cp == i32(0x1E9E)) => {
+      out.*.push(u8(0x73)); // s
+      out.*.push(u8(0x73)); // s
+      true
+    },
+    // İ (U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE) → i + combining dot above
+    (cp == i32(0x0130)) => {
+      out.*.push(u8(0x69)); // i
+      // U+0307 COMBINING DOT ABOVE
+      _encode_utf8(i32(0x0307), out);
+      true
+    },
+    true => false
+  )
+});
+
+// Special case folding: codepoints that expand to multiple codepoints
+// when uppercased.
+_special_to_upper :: (fn(cp: i32, out: *(ArrayList(u8))) -> bool)({
+  cond(
+    // ß (U+00DF LATIN SMALL LETTER SHARP S) → SS
+    (cp == i32(0x00DF)) => {
+      out.*.push(u8(0x53)); // S
+      out.*.push(u8(0x53)); // S
+      true
+    },
+    // ff (U+FB00 LATIN SMALL LIGATURE FF) → FF
+    (cp == i32(0xFB00)) => {
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x46)); // F
+      true
+    },
+    // fi (U+FB01 LATIN SMALL LIGATURE FI) → FI
+    (cp == i32(0xFB01)) => {
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x49)); // I
+      true
+    },
+    // fl (U+FB02 LATIN SMALL LIGATURE FL) → FL
+    (cp == i32(0xFB02)) => {
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x4C)); // L
+      true
+    },
+    // ffi (U+FB03 LATIN SMALL LIGATURE FFI) → FFI
+    (cp == i32(0xFB03)) => {
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x49)); // I
+      true
+    },
+    // ffl (U+FB04 LATIN SMALL LIGATURE FFL) → FFL
+    (cp == i32(0xFB04)) => {
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x46)); // F
+      out.*.push(u8(0x4C)); // L
+      true
+    },
+    // ſt (U+FB05 LATIN SMALL LIGATURE LONG S T) → ST
+    (cp == i32(0xFB05)) => {
+      out.*.push(u8(0x53)); // S
+      out.*.push(u8(0x54)); // T
+      true
+    },
+    // st (U+FB06 LATIN SMALL LIGATURE ST) → ST
+    (cp == i32(0xFB06)) => {
+      out.*.push(u8(0x53)); // S
+      out.*.push(u8(0x54)); // T
+      true
+    },
+    true => false
+  )
+});
+
+// Convert a String to lowercase using Unicode-aware case mapping.
+// Handles both ASCII and non-ASCII codepoints via C's towlower,
+// plus special multi-char expansions (e.g., ẞ → ss).
+unicode_to_lowercase :: (fn(input: String) -> String)({
+  (bytes : ArrayList(u8)) = input.as_bytes();
+  (out : ArrayList(u8)) = ArrayList(u8).with_capacity(bytes.len());
+  (i : usize) = usize(0);
+
+  while (i < bytes.len()), {
+    (b0 : i32) = i32(bytes.get(i).unwrap());
+    // Fast path for ASCII
+    if(((b0 & i32(0x80)) == i32(0)), {
+      if(((b0 >= i32(0x41)) && (b0 <= i32(0x5A))), {
+        out.push(u8((b0 + i32(0x20))));
+      }, {
+        out.push(u8(b0));
+      });
+      i = (i + usize(1));
+    }, {
+      // Multi-byte UTF-8: decode codepoint
+      (result : _DecodeResult) = _decode_utf8(bytes, i);
+      (cp : i32) = result.codepoint;
+      (len : usize) = result.bytes_consumed;
+
+      // Try special case folding first
+      if(!(_special_to_lower(cp, (&out))), {
+        // Use C towlower for standard Unicode lowercase
+        (lower : i32) = i32(towlower(cp));
+        _encode_utf8(lower, (&out));
+      });
+
+      i = (i + len);
+    });
+  };
+
+  String.from_bytes(out)
+});
+
+// Convert a String to uppercase using Unicode-aware case mapping.
+// Handles both ASCII and non-ASCII codepoints via C's towupper,
+// plus special multi-char expansions (e.g., ß → SS, ligatures).
+unicode_to_uppercase :: (fn(input: String) -> String)({
+  (bytes : ArrayList(u8)) = input.as_bytes();
+  (out : ArrayList(u8)) = ArrayList(u8).with_capacity(bytes.len());
+  (i : usize) = usize(0);
+
+  while (i < bytes.len()), {
+    (b0 : i32) = i32(bytes.get(i).unwrap());
+    // Fast path for ASCII
+    if(((b0 & i32(0x80)) == i32(0)), {
+      if(((b0 >= i32(0x61)) && (b0 <= i32(0x7A))), {
+        out.push(u8((b0 - i32(0x20))));
+      }, {
+        out.push(u8(b0));
+      });
+      i = (i + usize(1));
+    }, {
+      // Multi-byte UTF-8: decode codepoint
+      (result : _DecodeResult) = _decode_utf8(bytes, i);
+      (cp : i32) = result.codepoint;
+      (len : usize) = result.bytes_consumed;
+
+      // Try special case folding first
+      if(!(_special_to_upper(cp, (&out))), {
+        // Use C towupper for standard Unicode uppercase
+        (upper : i32) = i32(towupper(cp));
+        _encode_utf8(upper, (&out));
+      });
+
+      i = (i + len);
+    });
+  };
+
+  String.from_bytes(out)
+});
+
+export unicode_to_lowercase, unicode_to_uppercase;