@shd101wyy/yo 0.1.28 → 0.1.30

package/scripts/wrap-extern-calls.ts

@@ -0,0 +1,161 @@
+/**
+ * plans/EXTERN_UNSAFE_WRAP.md Phase C — apply `unsafe(...)` wraps at
+ * every extern "c" call site collected by the evaluator's dump mode.
+ *
+ * Workflow:
+ *   1. YO_EXTERN_WRAP_DUMP_FILE=/tmp/extern_wraps.json ./yo-cli check ./std
+ *      (the evaluator records each unwrapped call site as JSON)
+ *   2. bun run scripts/wrap-extern-calls.ts /tmp/extern_wraps.json --write
+ *      (this script reads the JSON and applies precise wraps)
+ *
+ * Dry-run by default. Pass `--write` to apply changes.
+ */
+import { readFileSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+type Site = {
+  modulePath: string;
+  characterOffset: number;
+  calleeName: string;
+};
+
+function findMatchingClose(src: string, openIdx: number): number {
+  if (src[openIdx] !== "(") {
+    throw new Error(`expected '(' at offset ${openIdx}, got '${src[openIdx]}'`);
+  }
+  let depth = 0;
+  for (let i = openIdx; i < src.length; i++) {
+    const c = src[i];
+    if (c === "(") depth++;
+    else if (c === ")") {
+      depth--;
+      if (depth === 0) return i;
+    }
+  }
+  throw new Error(`unmatched '(' at offset ${openIdx}`);
+}
+
+function alreadyWrapped(src: string, callStart: number): boolean {
+  // Look back from callStart past whitespace; check for `unsafe(` immediately
+  // before.
+  let j = callStart - 1;
+  while (j >= 0 && (src[j] === " " || src[j] === "\t" || src[j] === "\n")) {
+    j--;
+  }
+  if (j < 6) return false;
+  return src.slice(j - 6, j + 1) === "unsafe(";
+}
+
+function findCallOpenParen(src: string, nameStart: number): number {
+  // Scan forward from `nameStart` past the function name to find the `(`.
+  // The function name is a sequence of identifier chars; the `(` follows
+  // (optionally with whitespace).
+  let i = nameStart;
+  while (i < src.length && /[A-Za-z0-9_]/.test(src[i]!)) i++;
+  while (i < src.length && /\s/.test(src[i]!)) i++;
+  if (src[i] !== "(") {
+    throw new Error(
+      `expected '(' after identifier starting at ${nameStart}, got '${src[i]}' at ${i}`
+    );
+  }
+  return i;
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const jsonPath = args[0];
+  const write = args.includes("--write");
+  if (!jsonPath) {
+    console.error(
+      "usage: bun run scripts/wrap-extern-calls.ts <json-path> [--write]"
+    );
+    process.exit(1);
+  }
+  const sites: Site[] = JSON.parse(readFileSync(jsonPath, "utf8"));
+
+  // Dedupe by (modulePath, characterOffset)
+  const seen = new Set<string>();
+  const unique: Site[] = [];
+  for (const s of sites) {
+    const k = `${s.modulePath}@${s.characterOffset}`;
+    if (seen.has(k)) continue;
+    seen.add(k);
+    unique.push(s);
+  }
+
+  // Group by file, sort offsets descending so edits don't shift later
+  // offsets in the same file.
+  const byFile = new Map<string, Site[]>();
+  for (const s of unique) {
+    const filePath = fileURLToPath(s.modulePath);
+    if (!byFile.has(filePath)) byFile.set(filePath, []);
+    byFile.get(filePath)!.push(s);
+  }
+  for (const list of byFile.values()) {
+    list.sort((a, b) => b.characterOffset - a.characterOffset);
+  }
+
+  let totalWraps = 0;
+  let totalSkipped = 0;
+  const fileStats: { file: string; wraps: number; skipped: number }[] = [];
+  for (const [filePath, list] of byFile) {
+    let src = readFileSync(filePath, "utf8");
+    let wraps = 0;
+    let skipped = 0;
+    for (const s of list) {
+      if (alreadyWrapped(src, s.characterOffset)) {
+        skipped++;
+        continue;
+      }
+      let openParen: number;
+      try {
+        openParen = findCallOpenParen(src, s.characterOffset);
+      } catch (e) {
+        console.warn(
+          `skip ${filePath}@${s.characterOffset} (${s.calleeName}): ${(e as Error).message}`
+        );
+        skipped++;
+        continue;
+      }
+      let close: number;
+      try {
+        close = findMatchingClose(src, openParen);
+      } catch (e) {
+        console.warn(
+          `skip ${filePath}@${s.characterOffset} (${s.calleeName}): ${(e as Error).message}`
+        );
+        skipped++;
+        continue;
+      }
+      src =
+        src.slice(0, s.characterOffset) +
+        "unsafe(" +
+        src.slice(s.characterOffset, close + 1) +
+        ")" +
+        src.slice(close + 1);
+      wraps++;
+    }
+    if (wraps > 0 && write) {
+      writeFileSync(filePath, src);
+    }
+    if (wraps + skipped > 0) {
+      fileStats.push({ file: filePath, wraps, skipped });
+      totalWraps += wraps;
+      totalSkipped += skipped;
+    }
+  }
+
+  fileStats.sort((a, b) => b.wraps - a.wraps);
+  for (const s of fileStats) {
+    console.log(
+      `${s.file}: ${s.wraps} wrap${s.wraps === 1 ? "" : "s"}` +
+        (s.skipped ? `, ${s.skipped} skipped` : "")
+    );
+  }
+  console.log(
+    `\n${write ? "Applied" : "Would apply"} ${totalWraps} wrap(s) across ${fileStats.length} file(s); ${totalSkipped} skipped.`
+  );
+  if (!write) console.log("Re-run with --write to apply.");
+}
+
+main();

package/std/alg/hash.yo

@@ -7,11 +7,12 @@ FNV_OFFSET_BASIS :: u64(14695981039346656037);
 /// FNV prime constant (0x100000001b3).
 FNV_PRIME :: u64(1099511628211);
 /// Hash a byte sequence using FNV-1a.
-fnv1a_hash_bytes :: (fn(bytes : *(u8), length : usize) -> u64)({
+fnv1a_hash_bytes :: (fn(bytes : Slice(u8)) -> u64)({
   hash := FNV_OFFSET_BASIS;
+  length := bytes.len();
   i := usize(0);
   while(i < length, i = (i + usize(1)), {
-    byte := (bytes &+ i).*;
+    byte := bytes(i);
     xor := (hash ^ u64(byte));
     hash = (xor * FNV_PRIME);
   });

package/std/allocator.yo

@@ -1,4 +1,5 @@
 //! Memory allocation abstractions and global allocator interface.
+pragma(Pragma.AllowUnsafe);
 { SIZE_MAX } :: import("./libc/stdint.yo");
 /// Memory allocation error variants.
 AllocError :: enum(
@@ -39,15 +40,15 @@ size_would_overflow ::
 /// Implement this trait to provide custom allocation strategies.
 CustomAllocator :: trait(
   /// Allocate memory for `count` items of type T.
-  alloc : (fn(self : *(Self), comptime(T) : Type, count : usize) -> Result(?*(T), AllocError)),
+  alloc : (fn(ref(self) : Self, comptime(T) : Type, count : usize) -> Result(?*(T), AllocError)),
   /// Allocate zeroed memory for `count` items of type T.
-  alloc_zeroed : (fn(self : *(Self), comptime(T) : Type, count : usize) -> Result(?*(T), AllocError)),
+  alloc_zeroed : (fn(ref(self) : Self, comptime(T) : Type, count : usize) -> Result(?*(T), AllocError)),
   /// Allocate aligned memory for `count` items of type T with the given alignment.
-  alloc_aligned : (fn(self : *(Self), comptime(T) : Type, count : usize, alignment : usize) -> Result(?*(T), AllocError)),
+  alloc_aligned : (fn(ref(self) : Self, comptime(T) : Type, count : usize, alignment : usize) -> Result(?*(T), AllocError)),
   /// Reallocate memory to `new_count` items, preserving existing data.
-  realloc : (fn(forall(T : Type), self : *(Self), old_ptr : ?*(T), new_count : usize) -> Result(?*(T), AllocError)),
+  realloc : (fn(forall(T : Type), ref(self) : Self, old_ptr : ?*(T), new_count : usize) -> Result(?*(T), AllocError)),
   /// Deallocate memory pointed to by `ptr`.
-  dealloc : (fn(forall(T : Type), self : *(Self), ptr : *(T)) -> Result(unit, AllocError))
+  dealloc : (fn(forall(T : Type), ref(self) : Self, ptr : *(T)) -> Result(unit, AllocError))
 );
 /// Type-erased dynamic allocator wrapping any `CustomAllocator` implementation.
 Allocator :: Dyn(CustomAllocator);

package/std/async.yo

@@ -1,8 +1,8 @@
 //! Async utilities for cooperative task scheduling.
 /// Suspend the current async task, allowing other tasks to run on the event loop.
 /// Implemented as an immediately-completing async block that forces a suspension point.
-yield :: (fn(using(io : IO)) -> Impl(Future(unit)))(
-  io.async(() => {
+yield :: (fn(io : Io) -> Impl(Future(unit)))(
+  io.async((io : Io) => {
     return(());
   })
 );

package/std/collections/array_list.yo

@@ -1,4 +1,5 @@
 //! Dynamic array (vector) with amortized O(1) push and O(1) indexed access.
+pragma(Pragma.AllowUnsafe);
 { GlobalAllocator, AllocError } :: import("../allocator.yo");
 { memmove, memcpy, memset } :: import("../libc/string.yo");
 { malloc, calloc, realloc, free, aligned_alloc } :: GlobalAllocator;
@@ -303,7 +304,12 @@ impl(
                         bytes_to_move := (elements_after * element_size);
                         dst_void := *(void)(dst_ptr);
                         src_void := *(void)(src_ptr);
-                        memmove(dst_void, src_void, bytes_to_move);
+                        // SAFETY: dst_ptr and src_ptr both point into self._ptr's
+                        // contiguous element buffer. `elements_after = self._length - (start + actual_count)`
+                        // counts elements still in-bounds after the remove range,
+                        // so `bytes_to_move = elements_after * sizeof(T)` stays within
+                        // the allocated buffer. memmove handles overlap.
+                        unsafe(memmove(dst_void, src_void, bytes_to_move));
                       }
                     );
                   },
@@ -429,7 +435,7 @@ impl(
           self._ptr,
           .Some(dst_base) => {
             dst := (*(void))(dst_base &+ self._length);
-            _ := memcpy(dst, (*(void))(src), count * sizeof(T));
+            _ := unsafe(memcpy(dst, (*(void))(src), count * sizeof(T)));
             self._length = (self._length + count);
           },
           .None => panic("ArrayList.extend_from_ptr: no ptr after ensure_total_capacity")
@@ -454,7 +460,7 @@ impl(
       self._ptr,
       .None => (),
       .Some(_ptr) => {
-        _ := memset((*(void))(_ptr), byte_val, self._length * sizeof(T));
+        _ := unsafe(memset((*(void))(_ptr), byte_val, self._length * sizeof(T)));
       }
     )
   ),
@@ -475,7 +481,7 @@ impl(
           .Some(_ptr) => {
             start := (*(void))(_ptr &+ self._length);
             fill_count := ((new_len - self._length) * sizeof(T));
-            _ := memset(start, byte_val, fill_count);
+            _ := unsafe(memset(start, byte_val, fill_count));
             self._length = new_len;
           },
           .None => panic("ArrayList.resize_with_byte: no ptr after ensure")
@@ -491,16 +497,38 @@ impl(
     /// The output type is the element type `T`.
     Output : T,
     /// Returns a pointer to the element at the given index. Panics if the index is out of bounds.
-    index : (fn(self : *(Self), idx : usize) -> *(Self.Output))({
-      assert(idx < self.*._length, "ArrayList: index out of bounds");
+    index : (fn(ref(self) : Self, idx : usize) -> *(Self.Output))({
+      assert(idx < self._length, "ArrayList: index out of bounds");
       match(
-        self.*._ptr,
+        self._ptr,
         .Some(_ptr) => (_ptr &+ idx),
         .None => panic("ArrayList: index on empty list")
       )
     })
   )
 );
+/// Indexable impl — projection-style indexing yielding `ref(T)`.
+/// See plans/ITERATOR_REDESIGN.md. Body wraps the pointer arithmetic
+/// in `unsafe(...)` to pass the flowability check.
+impl(
+  forall(T : Type),
+  ArrayList(T),
+  Indexable(usize)(
+    Element : T,
+    project : (fn(ref(self) : Self, pos : usize) -> ref(T))({
+      assert(pos < self._length, "ArrayList: project out of bounds");
+      match(
+        self._ptr,
+        // SAFETY: assert above bounds `pos < self._length` and the
+        // length invariant guarantees `pos < self._capacity`, so
+        // `_ptr &+ pos` stays inside the Rc-managed heap buffer
+        // for the duration of the borrow.
+        .Some(_ptr) => unsafe(_ptr &+ pos),
+        .None => panic("ArrayList: project on empty list")
+      )
+    })
+  )
+);
 impl(
   forall(T : Type),
   ArrayList(T),
@@ -539,7 +567,7 @@ impl(
   ArrayListIter(T),
   Iterator(
     Item : T,
-    next : (fn(self : *(Self)) -> Option(T))(
+    next : (fn(ref(self) : Self) -> Option(T))(
       cond(
         (self._index >= self._list._length) =>.None,
         true => {
@@ -558,34 +586,26 @@ impl(
     ArrayListIter(T)(_list : self, _index : usize(0))
   )
 );
-/**
-* Pointer iterator for ArrayList - yields pointers to elements (*(T))
-* Used by iter() method
-*/
-ArrayListIterPtr :: (fn(comptime(T) : Type) -> comptime(Type))(
-  struct(
-    _list : ArrayList(T),
-    _index : usize
-  )
+/// Position iterator for ArrayList — yields `usize` indices into the
+/// list. Used by `ArrayList.iter()`; the for-macro pairs each yielded
+/// position with `ArrayList.project(pos)` (Indexable from Phase C) to
+/// bind the body's `ref`-name. See plans/ITERATOR_REDESIGN.md.
+_ArrayListPosIter :: struct(
+  _index : usize,
+  _len : usize
 );
 impl(
-  forall(T : Type),
-  ArrayListIterPtr(T),
+  _ArrayListPosIter,
   Iterator(
-    Item : *(T),
-    next : (fn(self : *(Self)) -> Option(*(T)))(
+    Item : usize,
+    next : (fn(ref(self) : Self) -> Option(usize))(
       cond(
-        (self._index >= self._list._length) =>.None,
-        true =>
-          match(
-            self._list._ptr,
-            .Some(ptr) => {
-              element_ptr := (ptr &+ self._index);
-              self._index = (self._index + usize(1));
-              .Some(element_ptr)
-            },
-            .None =>.None
-          )
+        (self._index >= self._len) =>.None,
+        true => {
+          out := self._index;
+          self._index = (self._index + usize(1));
+          .Some(out)
+        }
       )
     )
   )
@@ -593,8 +613,8 @@ impl(
 impl(
   forall(T : Type),
   ArrayList(T),
-  iter : (fn(self : *(Self)) -> ArrayListIterPtr(T))(
-    ArrayListIterPtr(T)(_list : self.*, _index : usize(0))
+  iter : (fn(ref(self) : Self) -> _ArrayListPosIter)(
+    _ArrayListPosIter(_index : usize(0), _len : self._length)
   )
 );
 impl(
@@ -760,16 +780,16 @@ impl(
   where(T <: Clone),
   ArrayList(T),
   Clone(
-    clone : (fn(self : *(Self)) -> Self)({
-      n := self.*.len();
+    clone : (fn(ref(self) : Self) -> Self)({
+      n := self.len();
       result := Self.with_capacity(n);
       i := usize(0);
       while(i < n, {
-        item_opt := self.*.get(i);
+        item_opt := self.get(i);
         match(
           item_opt,
           .Some(item) => {
-            result.push((&(item)).clone());
+            result.push(item.clone());
           },
           .None => ()
         );
@@ -782,8 +802,7 @@ impl(
 export(
   ArrayList,
   ArrayListError,
-  ArrayListIter,
-  ArrayListIterPtr
+  ArrayListIter
 );
 // =============================================================================
 // array_list! literal macro

package/std/collections/btree_map.yo

@@ -10,6 +10,7 @@
 //! m.set(String.from("a"), 1);
 //! // Iteration order: a, b
 //! ```
+pragma(Pragma.AllowUnsafe);
 { ArrayList } :: import("./array_list");
 open(import("../string"));
 /// Key-value pair entry stored in a BTreeMap.
@@ -48,7 +49,7 @@ impl(
     hi := self._entries.len();
     while(lo < hi, {
       mid := (lo + ((hi - lo) / usize(2)));
-      entry_key := self._entries.get(mid).unwrap().key;
+      entry_key := self._entries(mid).key;
       cond(
         (entry_key < k) => {
           lo = (mid + usize(1));
@@ -67,7 +68,7 @@ impl(
   get : (fn(self : Self, k : K, where(K <: Ord(K))) -> ?(V))({
     r := self._find(k);
     cond(
-      r.found =>.Some(self._entries.get(r.idx).unwrap().value),
+      r.found =>.Some(self._entries(r.idx).value),
       true =>.None
     )
   }),
@@ -77,7 +78,7 @@ impl(
     r := self._find(k);
     cond(
       r.found => {
-        entry := self._entries.get(r.idx).unwrap();
+        entry := self._entries(r.idx);
         &(self._entries(r.idx)).* = BTreeEntry(K, V)(key : entry.key, value : v);
       },
       true => {
@@ -85,8 +86,8 @@ impl(
         self._entries.push(BTreeEntry(K, V)(key : k, value : v));
         pos := (self._entries.len() - usize(1));
         while(pos > r.idx, pos = (pos - usize(1)), {
-          curr := self._entries.get(pos).unwrap();
-          prev := self._entries.get(pos - usize(1)).unwrap();
+          curr := self._entries(pos);
+          prev := self._entries(pos - usize(1));
           &(self._entries(pos)).* = prev;
           &(self._entries(pos - usize(1))).* = curr;
         });
@@ -98,7 +99,7 @@ impl(
     r := self._find(k);
     cond(
       r.found => {
-        v := self._entries.get(r.idx).unwrap().value;
+        v := self._entries(r.idx).value;
         self._entries.remove(r.idx);
         .Some(v)
       },
@@ -109,14 +110,14 @@ impl(
   min : (fn(self : Self) -> ?(BTreeEntry(K, V)))(
     cond(
       (self._entries.len() == usize(0)) =>.None,
-      true =>.Some(self._entries.get(usize(0)).unwrap())
+      true =>.Some(self._entries(usize(0)))
     )
   ),
   // Return the entry with the largest key, or .None.
   max : (fn(self : Self) -> ?(BTreeEntry(K, V)))(
     cond(
       (self._entries.len() == usize(0)) =>.None,
-      true =>.Some(self._entries.get(self._entries.len() - usize(1)).unwrap())
+      true =>.Some(self._entries(self._entries.len() - usize(1)))
     )
   )
 );
@@ -134,11 +135,11 @@ impl(
   BTreeMapIter(K, V),
   Iterator(
     Item : BTreeEntry(K, V),
-    next : (fn(self : *(Self)) -> Option(BTreeEntry(K, V)))(
+    next : (fn(ref(self) : Self) -> Option(BTreeEntry(K, V)))(
       cond(
         (self._index >= self._entries.len()) =>.None,
         true => {
-          entry := self._entries.get(self._index).unwrap();
+          entry := self._entries(self._index);
           self._index = (self._index + usize(1));
           .Some(entry)
         }
@@ -167,7 +168,7 @@ impl(
   BTreeMapIterPtr(K, V),
   Iterator(
     Item : *(BTreeEntry(K, V)),
-    next : (fn(self : *(Self)) -> Option(*(BTreeEntry(K, V))))(
+    next : (fn(ref(self) : Self) -> Option(*(BTreeEntry(K, V))))(
       cond(
         (self._index >= self._entries.len()) =>.None,
         true =>
@@ -187,8 +188,8 @@ impl(
 impl(
   forall(K : Type, V : Type),
   BTreeMap(K, V),
-  iter : (fn(self : *(Self)) -> BTreeMapIterPtr(K, V))(
-    BTreeMapIterPtr(K, V)(_entries : self.*._entries, _index : usize(0))
+  iter : (fn(self : Self) -> BTreeMapIterPtr(K, V))(
+    BTreeMapIterPtr(K, V)(_entries : self._entries, _index : usize(0))
   )
 );
 /**
@@ -204,7 +205,7 @@ impl(
   BTreeMapKeys(K, V),
   Iterator(
     Item : K,
-    next : (fn(self : *(Self)) -> Option(K))(
+    next : (fn(ref(self) : Self) -> Option(K))(
       match(
         self._inner.next(),
         .None =>.None,
@@ -233,7 +234,7 @@ impl(
   BTreeMapValues(K, V),
   Iterator(
     Item : V,
-    next : (fn(self : *(Self)) -> Option(V))(
+    next : (fn(ref(self) : Self) -> Option(V))(
       match(
         self._inner.next(),
         .None =>.None,
@@ -254,10 +255,10 @@ impl(
   BTreeMap(K, V),
   Index(K)(
     Output : V,
-    index : (fn(self : *(Self), idx : K, where(K <: Ord(K))) -> *(Self.Output))({
-      r := self.*._find(idx);
+    index : (fn(ref(self) : Self, idx : K, where(K <: Ord(K))) -> *(Self.Output))({
+      r := self._find(idx);
       assert(r.found, "BTreeMap: key not found");
-      entry_ptr := &(self.*._entries(r.idx));
+      entry_ptr := &(self._entries(r.idx));
       &(entry_ptr.*.value)
     })
   )

package/std/collections/deque.yo

@@ -10,6 +10,7 @@
 //! d.push_front(0);
 //! x := d.pop_front().unwrap();  // 0
 //! ```
+pragma(Pragma.AllowUnsafe);
 { GlobalAllocator, AllocError } :: import("../allocator");
 { malloc, realloc, free } :: GlobalAllocator;
 /// Circular buffer supporting efficient push/pop at both front and back.
@@ -188,7 +189,7 @@ impl(
   DequeIter(T),
   Iterator(
     Item : T,
-    next : (fn(self : *(Self)) -> Option(T))(
+    next : (fn(ref(self) : Self) -> Option(T))(
       cond(
         (self._remaining == usize(0)) =>.None,
         true => {
@@ -227,7 +228,7 @@ impl(
   DequeIterPtr(T),
   Iterator(
     Item : *(T),
-    next : (fn(self : *(Self)) -> Option(*(T)))(
+    next : (fn(ref(self) : Self) -> Option(*(T)))(
       cond(
         (self._remaining == usize(0)) =>.None,
         true => {
@@ -245,8 +246,8 @@ impl(
 impl(
   forall(T : Type),
   Deque(T),
-  iter : (fn(self : *(Self)) -> DequeIterPtr(T))(
-    DequeIterPtr(T)(_deque : self.*, _pos : usize(0), _remaining : self.*._len)
+  iter : (fn(self : Self) -> DequeIterPtr(T))(
+    DequeIterPtr(T)(_deque : self, _pos : usize(0), _remaining : self._len)
   )
 );
 impl(
@@ -254,10 +255,10 @@ impl(
   Deque(T),
   Index(usize)(
     Output : T,
-    index : (fn(self : *(Self), idx : usize) -> *(Self.Output))({
-      assert(idx < self.*._len, "Deque: index out of bounds");
-      buf := self.*._buf.unwrap();
-      physical_idx := ((self.*._head + idx) % self.*._capacity);
+    index : (fn(ref(self) : Self, idx : usize) -> *(Self.Output))({
+      assert(idx < self._len, "Deque: index out of bounds");
+      buf := self._buf.unwrap();
+      physical_idx := ((self._head + idx) % self._capacity);
       buf &+ physical_idx
     })
   )