@shd101wyy/yo 0.1.28 → 0.1.30

package/scripts/migrate-self-ptr.ts

@@ -0,0 +1,220 @@
+/**
+ * Migrate `(self : *(Self))` method signatures to `(inout(self) : Self)`
+ * inside `impl(...)` blocks, and rewrite `self.*` references in the
+ * matching method bodies to `self`.
+ *
+ * Operates on a single file (passed as argv[2]). The script is
+ * intentionally narrow:
+ *  - Only rewrites function signatures of the exact shape
+ *    `(self : *(Self), …) -> …` or `(self : *(Self)) -> …`.
+ *  - Only rewrites `self.*` *inside the body* of one of those
+ *    methods, found by walking balanced parens after the signature.
+ *  - `self.*` outside any such method body — for example in helper
+ *    functions that operate on a `*(SomeType)` parameter — is left
+ *    alone.
+ *
+ * Patterns deliberately NOT rewritten (need manual review):
+ *  - `(&(self.*.field))…` array-element mutation patterns
+ *  - `(other : *(SomeType))` non-self pointer parameters
+ *  - `self.*.field = …` writes (these are valid `self.field = …`
+ *    after migration, but worth eyeballing the surrounding context)
+ *
+ * Usage:
+ *   bun run scripts/migrate-self-ptr.ts <file>          # dry-run
+ *   bun run scripts/migrate-self-ptr.ts <file> --write  # apply
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import * as path from "node:path";
+
+interface RewriteResult {
+  result: string;
+  sigsRewritten: number;
+  derefRewrites: number;
+}
+
+function migrate(content: string): RewriteResult {
+  const out: string[] = [];
+  let i = 0;
+  let sigsRewritten = 0;
+  let derefRewrites = 0;
+
+  // Match `(self : *(Self)` or `(self : *(Self),` — the signature
+  // start. We need to be inside a function-type expression, but it's
+  // hard to verify that without a real parser; rely on the literal
+  // shape, which is uncommon outside this context.
+  const sigStart = /\(self : \*\(Self\)/g;
+
+  // Heuristic: skip `*(Self)` matches inside Iterator or Index trait
+  // impls. Those trait declarations specify `*(Self)` and changing
+  // the impl alone would break trait conformance. We detect the
+  // enclosing trait by scanning backwards from each match for the
+  // nearest unmatched `Iterator(` or `Index(...)(` opener.
+  const isInsideTraitImpl = (matchIdx: number): boolean => {
+    // Scan backwards looking for `Iterator(` or `Index(...)(` at a
+    // depth where its `(` is still open at matchIdx. We use a
+    // simple paren-balance check.
+    let depth = 0;
+    for (let j = matchIdx - 1; j >= 0; j--) {
+      const ch = content[j]!;
+      if (ch === ")") depth++;
+      else if (ch === "(") {
+        if (depth === 0) {
+          // This `(` is one whose scope contains matchIdx. Check
+          // what immediately precedes it for a trait name.
+          const before = content.slice(Math.max(0, j - 30), j);
+          if (/\bIterator\s*$/.test(before)) return true;
+          if (/\bIndex\s*\([^()]*\)\s*$/.test(before)) return true;
+          // Not Iterator/Index — keep scanning outward.
+        } else {
+          depth--;
+        }
+      }
+    }
+    return false;
+  };
+
+  while (i < content.length) {
+    sigStart.lastIndex = i;
+    const m = sigStart.exec(content);
+    if (!m) {
+      out.push(content.slice(i));
+      break;
+    }
+    const startIdx = m.index;
+
+    // Skip Iterator/Index trait impl methods — they're declared with
+    // `*(Self)` in the trait and migrating only the impl would break
+    // conformance. The trait declarations themselves stay verbatim.
+    if (isInsideTraitImpl(startIdx)) {
+      out.push(content.slice(i, startIdx + m[0].length));
+      i = startIdx + m[0].length;
+      continue;
+    }
+
+    out.push(content.slice(i, startIdx));
+
+    // The match starts with `(` of the param list. Walk balanced parens
+    // to find the end of the param-list `)`. We DON'T rewrite the
+    // body's self.* until we find the method body, which is the
+    // *following* parenthesized expression.
+    let j = startIdx;
+    let depth = 0;
+    while (j < content.length) {
+      const ch = content[j]!;
+      if (ch === "(") depth++;
+      else if (ch === ")") {
+        depth--;
+        if (depth === 0) {
+          j++;
+          break;
+        }
+      }
+      j++;
+    }
+    // We've now consumed the `(self : *(Self), …)` param list. Emit
+    // the rewritten signature.
+    const sigText = content.slice(startIdx, j);
+    const newSig = sigText.replace("(self : *(Self)", "(inout(self) : Self");
+    out.push(newSig);
+    sigsRewritten++;
+
+    // After the param list, we expect ` -> ReturnType)( body )`.
+    // We don't care to fully parse — we just keep walking until we
+    // find the next top-level `)`, which closes the `fn(...)` type,
+    // then the next `(` opens the body. Inside the body (balanced
+    // parens) we rewrite `self.*` → `self`.
+    //
+    // Practically: keep emitting content unchanged until we hit a
+    // matching close-paren depth at the level of the `fn` type
+    // expression, then walk into the body.
+    //
+    // Simpler approach: find the next `)` from current position that
+    // closes the surrounding fn-type expression. We know the
+    // pattern: `(fn(self : *(Self), …) -> RETURN_TYPE)`. We've consumed
+    // up through the params; now expect ` -> RETURN_TYPE)`.
+    let k = j;
+    // Walk past ` -> ReturnType)` — handle nested parens in the
+    // return type.
+    let returnDepth = 0;
+    while (k < content.length) {
+      const ch = content[k]!;
+      if (ch === "(") returnDepth++;
+      else if (ch === ")") {
+        if (returnDepth === 0) {
+          k++;
+          break;
+        }
+        returnDepth--;
+      }
+      k++;
+    }
+    // Emit ` -> RETURN_TYPE)` unchanged.
+    out.push(content.slice(j, k));
+
+    // Now the body. Skip whitespace.
+    while (k < content.length && /\s/.test(content[k]!)) {
+      out.push(content[k]!);
+      k++;
+    }
+    // The body should start with `(`. If not, give up rewriting this
+    // method's `self.*` references (signature still flipped, but
+    // body left alone — manual review needed).
+    if (content[k] !== "(") {
+      i = k;
+      continue;
+    }
+    // Walk balanced parens for the body.
+    const bodyStart = k;
+    let bodyDepth = 0;
+    while (k < content.length) {
+      const ch = content[k]!;
+      if (ch === "(") bodyDepth++;
+      else if (ch === ")") {
+        bodyDepth--;
+        if (bodyDepth === 0) {
+          k++;
+          break;
+        }
+      }
+      k++;
+    }
+    const body = content.slice(bodyStart, k);
+    // Rewrite `self.*` → `self` inside this method's body only.
+    const rewrittenBody = body.replace(/self\.\*/g, () => {
+      derefRewrites++;
+      return "self";
+    });
+    out.push(rewrittenBody);
+
+    i = k;
+  }
+
+  return { result: out.join(""), sigsRewritten, derefRewrites };
+}
+
+function main(): void {
+  const args = process.argv.slice(2);
+  const write = args.includes("--write");
+  const file = args.find((a) => !a.startsWith("--"));
+  if (!file) {
+    console.error(
+      "Usage: bun run scripts/migrate-self-ptr.ts <file> [--write]"
+    );
+    process.exit(1);
+  }
+  const abs = path.resolve(file);
+  const content = readFileSync(abs, "utf-8");
+  const { result, sigsRewritten, derefRewrites } = migrate(content);
+  if (result === content) {
+    console.log(`no changes: ${file}`);
+    return;
+  }
+  console.log(
+    `${write ? "migrated" : "would migrate"}: ${file} ` +
+      `(${sigsRewritten} signatures, ${derefRewrites} self.* rewrites)`
+  );
+  if (write) writeFileSync(abs, result, "utf-8");
+}
+
+main();

package/scripts/migrate-skip-pragmas.ts

@@ -0,0 +1,109 @@
+/**
+ * One-shot migration: convert `// @skip_*` comment directives to
+ * `pragma(Pragma.Skip*);` calls.
+ *
+ * Only line-leading comment directives are migrated — directives
+ * inside string literals or doc-comments are left alone. Each
+ * matching line is replaced by the corresponding pragma call,
+ * preserving any trailing "— rationale" text as a leading comment.
+ *
+ * Usage:
+ *   bun run scripts/migrate-skip-pragmas.ts             # dry-run
+ *   bun run scripts/migrate-skip-pragmas.ts --write     # apply
+ */
+
+import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import * as path from "node:path";
+
+const ROOTS = ["tests", "std", "yo-self", "src/tests"];
+
+const MAPPING: Array<{ pattern: RegExp; variant: string }> = [
+  // Order matters: longer/more-specific first so the broader
+  // `@skip_wasm` doesn't shadow `@skip_wasm32-...`.
+  {
+    pattern: /^(\s*)\/\/\s*@skip_wasm32-emscripten\b(.*)$/,
+    variant: "SkipWasm32Emscripten",
+  },
+  {
+    pattern: /^(\s*)\/\/\s*@skip_wasm32-wasi\b(.*)$/,
+    variant: "SkipWasm32Wasi",
+  },
+  { pattern: /^(\s*)\/\/\s*@skip_wasm\b(.*)$/, variant: "SkipWasm" },
+  { pattern: /^(\s*)\/\/\s*@skip_prelude\b(.*)$/, variant: "SkipPrelude" },
+];
+
+function* walk(dir: string): Generator<string> {
+  for (const entry of readdirSync(dir)) {
+    const fullPath = path.join(dir, entry);
+    const st = statSync(fullPath);
+    if (st.isDirectory()) {
+      yield* walk(fullPath);
+    } else if (st.isFile() && entry.endsWith(".yo")) {
+      yield fullPath;
+    }
+  }
+}
+
+function migrateFile(
+  filePath: string,
+  write: boolean
+): {
+  changed: boolean;
+  matches: number;
+} {
+  const original = readFileSync(filePath, "utf-8");
+  const lines = original.split("\n");
+  let matches = 0;
+  const out: string[] = [];
+  for (const line of lines) {
+    let migrated: string | null = null;
+    for (const { pattern, variant } of MAPPING) {
+      const m = line.match(pattern);
+      if (m) {
+        const leading = m[1] ?? "";
+        const trailing = (m[2] ?? "").trim();
+        const rationale = trailing
+          ? ` // ${trailing.replace(/^[—-]\s*/, "")}`
+          : "";
+        migrated = `${leading}pragma(Pragma.${variant});${rationale}`;
+        matches++;
+        break;
+      }
+    }
+    out.push(migrated ?? line);
+  }
+  if (matches === 0) return { changed: false, matches: 0 };
+  const result = out.join("\n");
+  if (write && result !== original) {
+    writeFileSync(filePath, result, "utf-8");
+  }
+  return { changed: result !== original, matches };
+}
+
+function main(): void {
+  const write = process.argv.includes("--write");
+  let totalFiles = 0;
+  let totalMatches = 0;
+  for (const root of ROOTS) {
+    const absRoot = path.resolve(root);
+    for (const filePath of walk(absRoot)) {
+      const { changed, matches } = migrateFile(filePath, write);
+      if (changed) {
+        totalFiles++;
+        totalMatches += matches;
+        console.log(
+          `${write ? "migrated" : "would migrate"}: ${path.relative(
+            process.cwd(),
+            filePath
+          )} (${matches} match${matches === 1 ? "" : "es"})`
+        );
+      }
+    }
+  }
+  console.log(
+    `\n${write ? "Migrated" : "Would migrate"} ${totalMatches} directives across ${totalFiles} file(s).`
+  );
+  if (!write) console.log("(dry-run — pass --write to apply)");
+}
+
+main();

package/scripts/migrate-tostring.ts

@@ -0,0 +1,134 @@
+/**
+ * One-shot migration: ToString trait impls from `(self : *(Self))` to
+ * `(inout(self) : Self)`. Phase D continuation of plans/MEMORY_SAFETY.md.
+ *
+ * Two pattern rewrites:
+ *
+ * 1. The impl signature:
+ *      to_string : (fn(self : *(Self)) -> String)({ ... self.* ... })
+ *    →
+ *      to_string : (fn(inout(self) : Self) -> String)({ ... self ... })
+ *
+ *    Replaces `self.*` inside ToString impl bodies with `self`. Outside
+ *    ToString impls (or in any other context), `self.*` is left alone.
+ *
+ * 2. Explicit caller patterns `(&(x)).to_string()` → `x.to_string()`.
+ *
+ * Usage:
+ *   bun run scripts/migrate-tostring.ts             # dry-run
+ *   bun run scripts/migrate-tostring.ts --write     # apply
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import * as path from "node:path";
+
+const FILES = [
+  "std/log.yo",
+  "std/fmt/to_string.yo",
+  "std/time/datetime.yo",
+  "std/time/duration.yo",
+  "std/testing/bench.yo",
+  "std/string/string_builder.yo",
+  "yo-self/parser.yo",
+  "yo-self/lexer.yo",
+  "yo-self/error.yo",
+];
+
+/**
+ * Find all to_string impl bodies in the file and rewrite each from
+ * `(self : *(Self)) -> String)({ ...body... })` to `(inout(self) :
+ * Self) -> String)({ ...body... })`, where `body` has `self.*`
+ * replaced by `self`. Only rewrites bodies that begin with the exact
+ * to_string signature we care about; everything else is preserved
+ * byte-for-byte.
+ */
+function migrateContent(content: string): { result: string; changed: boolean } {
+  let changed = false;
+  // Pattern: `to_string : (fn(self : *(Self)) -> String)` followed by a
+  // body. We need to find the balanced `(...)` of the body and rewrite
+  // `self.*` → `self` inside it.
+  const signature = "to_string : (fn(self : *(Self)) -> String)";
+  const newSignature = "to_string : (fn(inout(self) : Self) -> String)";
+  const out: string[] = [];
+  let i = 0;
+  while (i < content.length) {
+    const idx = content.indexOf(signature, i);
+    if (idx === -1) {
+      out.push(content.slice(i));
+      break;
+    }
+    // Push everything before the match
+    out.push(content.slice(i, idx));
+    out.push(newSignature);
+    let j = idx + signature.length;
+    // Find the body — could be `({...})` or `(expr)` immediately after.
+    // Skip whitespace.
+    while (j < content.length && /\s/.test(content[j]!)) {
+      out.push(content[j]!);
+      j++;
+    }
+    if (content[j] !== "(") {
+      // Unexpected shape; bail out for this match — preserve the rest verbatim.
+      out.push(content.slice(j));
+      changed = true;
+      break;
+    }
+    // Walk matching parens; rewrite self.* → self inside the body.
+    let depth = 0;
+    const bodyStart = j;
+    while (j < content.length) {
+      const ch = content[j]!;
+      if (ch === "(") depth++;
+      else if (ch === ")") {
+        depth--;
+        if (depth === 0) {
+          j++;
+          break;
+        }
+      }
+      j++;
+    }
+    const body = content.slice(bodyStart, j);
+    // Replace `self.*` with `self` in the impl body. Be careful: do
+    // not replace `self.*.field` patterns incorrectly. Just replace
+    // every occurrence — for trait impls the surrounding code has
+    // already been audited.
+    const rewrittenBody = body.replace(/self\.\*/g, "self");
+    out.push(rewrittenBody);
+    i = j;
+    changed = true;
+  }
+
+  let result = out.join("");
+
+  // Pattern 2: `(&(x)).to_string()` → `x.to_string()`. Conservative —
+  // only matches when `x` is a single identifier (possibly chained
+  // via `.`).
+  const callerPattern = /\(&\(([^()]+)\)\)\.to_string\(\)/g;
+  const before = result;
+  result = result.replace(callerPattern, "$1.to_string()");
+  if (result !== before) changed = true;
+
+  return { result, changed };
+}
+
+function main(): void {
+  const write = process.argv.includes("--write");
+  let totalChanged = 0;
+  for (const rel of FILES) {
+    const abs = path.resolve(rel);
+    const content = readFileSync(abs, "utf-8");
+    const { result, changed } = migrateContent(content);
+    if (changed && result !== content) {
+      totalChanged++;
+      console.log(`${write ? "migrated" : "would migrate"}: ${rel}`);
+      if (write) writeFileSync(abs, result, "utf-8");
+    }
+  }
+  console.log(
+    `\n${write ? "Migrated" : "Would migrate"} ${totalChanged} file(s).`
+  );
+  if (!write) console.log("(dry-run — pass --write to apply)");
+}
+
+main();

package/scripts/trim-pragma.ts

@@ -0,0 +1,130 @@
+#!/usr/bin/env bun
+/**
+ * Remove `pragma(Pragma.AllowUnsafe);` from files that don't actually
+ * need it — i.e., files that don't contain any `unsafe(...)`,
+ * `asm(...)`, or `extern(...)` calls.
+ *
+ * Phase C of plans/MEMORY_SAFETY.md added the pragma to every file
+ * under `std/`, `yo-self/`, and `tests/` mechanically. Many tests
+ * (and a few stdlib files) don't actually exercise raw-pointer
+ * machinery; the pragma was added defensively. Removing it from
+ * those files makes the test suite a better demonstration of
+ * "safe user code by default" and shrinks the auditable surface.
+ *
+ * Safety: we only consider a file "needs pragma" if it contains a
+ * direct `unsafe(`, `asm(`, or `extern(` call site (matched outside
+ * comments/strings, same heuristic as `yo unsafe-report`). If the
+ * file imports a stdlib module whose public API takes `*(T)`, the
+ * file still compiles in safe mode (per the current Phase C
+ * implementation — see "Known gaps" in plans/MEMORY_SAFETY.md).
+ *
+ * Usage:
+ *   bun scripts/trim-pragma.ts <file.yo> [more...]
+ *   bun scripts/trim-pragma.ts --dry-run <file.yo> [more...]
+ */
+
+import { readFileSync, writeFileSync, statSync } from "fs";
+import { argv } from "process";
+
+const dryRun = argv.includes("--dry-run");
+const files = argv.slice(2).filter((a) => a !== "--dry-run");
+
+const PRAGMA_LINE_RE = /^pragma\(Pragma\.AllowUnsafe\);\s*$/m;
+
+/**
+ * Strip line-comments and string literals from a line so that
+ * matches inside `"unsafe(...)"` or `// asm(...)` don't count.
+ */
+function strip(line: string): string {
+  const out: string[] = [];
+  let inStr: string | null = null;
+  let i = 0;
+  while (i < line.length) {
+    const ch = line[i]!;
+    if (!inStr && ch === "/" && line[i + 1] === "/") break;
+    if (inStr) {
+      if (ch === "\\") {
+        i += 2;
+        continue;
+      }
+      if (ch === inStr) inStr = null;
+      i++;
+      continue;
+    }
+    if (ch === '"' || ch === "`") {
+      inStr = ch;
+      i++;
+      continue;
+    }
+    out.push(ch);
+    i++;
+  }
+  return out.join("");
+}
+
+function fileNeedsPragma(src: string): { needs: boolean; reason?: string } {
+  const lines = src.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const s = strip(lines[i]!);
+    if (/\bunsafe\(/.test(s))
+      return { needs: true, reason: `unsafe(...) at line ${i + 1}` };
+    if (/(?:^|[^A-Za-z0-9_])asm\(/.test(s))
+      return { needs: true, reason: `asm(...) at line ${i + 1}` };
+    if (/(?:^|[^A-Za-z0-9_])extern\(/.test(s))
+      return { needs: true, reason: `extern(...) at line ${i + 1}` };
+    // Bare pointer-op sites that would fire the Phase A gate without
+    // a pragma. Without these checks we'd strip pragma from files
+    // like tests/ptr.test.yo that use `.*` / `&+` directly without
+    // wrapping in unsafe(...) (the pragma currently bypasses the
+    // gate so those work). Files like that should keep their pragma
+    // until/unless the pointer ops are wrapped in unsafe(...).
+    if (/\.\*(?:[^A-Za-z0-9_]|$)/.test(s))
+      return { needs: true, reason: `bare .* deref at line ${i + 1}` };
+    if (/(?:^|[^A-Za-z0-9_&])&\+|&-(?!=)|&\//.test(s))
+      return {
+        needs: true,
+        reason: `bare pointer arithmetic (&+/&-/&/) at line ${i + 1}`,
+      };
+  }
+  return { needs: false };
+}
+
+function processFile(file: string): "removed" | "kept" | "no-pragma" | "skip" {
+  try {
+    if (!statSync(file).isFile()) return "skip";
+  } catch {
+    return "skip";
+  }
+  const src = readFileSync(file, "utf8");
+  if (!PRAGMA_LINE_RE.test(src)) return "no-pragma";
+
+  const { needs, reason } = fileNeedsPragma(src);
+  if (needs) {
+    if (dryRun) console.log(`keep: ${file} (${reason})`);
+    return "kept";
+  }
+
+  // Remove the pragma line. Also remove a trailing blank line if it
+  // creates an awkward double-blank.
+  const next = src
+    .replace(/^pragma\(Pragma\.AllowUnsafe\);\n/m, "")
+    .replace(/^pragma\(Pragma\.AllowUnsafe\);\s*\n/m, "");
+
+  if (dryRun) {
+    console.log(`would-remove: ${file}`);
+    return "removed";
+  }
+
+  writeFileSync(file, next);
+  console.log(`removed: ${file}`);
+  return "removed";
+}
+
+const counts = { removed: 0, kept: 0, "no-pragma": 0, skip: 0 };
+for (const file of files) {
+  const r = processFile(file);
+  counts[r]++;
+}
+console.log(
+  `\nsummary: removed=${counts.removed}, kept=${counts.kept}, no-pragma=${counts["no-pragma"]}, skipped=${counts.skip}`
+);