@defold-typescript/library-types 0.20.6 → 0.20.8

package/scripts/parse-luals.ts

@@ -0,0 +1,383 @@
+/**
+ * A line-oriented reader for the LuaLS `---@` annotation dialect that druid-style
+ * pure-Lua libraries ship in place of a `.d.ts`. It populates a `LibraryModel`, a
+ * richer OOP shape than the flat `ApiModule` (`packages/types/src/api-doc.ts`):
+ * interfaces with methods/fields/generics/extends, aliases, and free module
+ * functions. Naming mirrors the flat model where it fits (`types: string[]`,
+ * `brief`, `isOptional`, `doc`) so the two read alike.
+ *
+ * Scope is parse-only: every LuaLS type expression is preserved as a raw token
+ * string, verbatim (`integer`, `string?`, `fun(self):number`, `table<K,V>`,
+ * `"a" | "b"`). Mapping those tokens to TypeScript is the next goal; this reader
+ * never rewrites, splits, or normalizes a type toward TS.
+ */
+
+export interface LibraryModel {
+  interfaces: LibraryInterface[];
+  aliases: LibraryAlias[];
+  moduleFunctions: LibraryMethod[];
+}
+
+export interface LibraryInterface {
+  name: string;
+  extends?: string;
+  generics: LibraryGeneric[];
+  fields: LibraryField[];
+  methods: LibraryMethod[];
+  brief: string;
+}
+
+export interface LibraryMethod {
+  name: string;
+  brief: string;
+  generics: LibraryGeneric[];
+  params: LibraryParam[];
+  returns: LibraryParam[];
+}
+
+export interface LibraryParam {
+  name: string;
+  types: string[];
+  doc: string;
+  isOptional: boolean;
+  isVararg: boolean;
+}
+
+export type LibraryFieldVisibility = "public" | "protected" | "private" | "package";
+
+export interface LibraryField {
+  name: string;
+  types: string[];
+  doc: string;
+  isOptional: boolean;
+  visibility?: LibraryFieldVisibility;
+}
+
+export interface LibraryGeneric {
+  name: string;
+  constraint?: string;
+}
+
+export interface LibraryAlias {
+  name: string;
+  types: string[];
+  doc: string;
+}
+
+interface Pending {
+  doc: string[];
+  params: LibraryParam[];
+  returns: LibraryParam[];
+  generics: LibraryGeneric[];
+}
+
+const emptyPending = (): Pending => ({ doc: [], params: [], returns: [], generics: [] });
+
+/**
+ * Read a single raw type token from the head of `rest`, honoring bracket depth so
+ * an inner space (`table<string, any>`, `fun(a, b): c`) does not end the token. The
+ * token ends at the first top-level whitespace. Returns the token and the trailing
+ * remainder (the human description). Never rewrites the token toward TS.
+ */
+function readTypeToken(rest: string): { type: string; rest: string } {
+  let depth = 0;
+  let i = 0;
+  for (; i < rest.length; i++) {
+    const c = rest[i];
+    if (c === "<" || c === "(" || c === "[" || c === "{") depth++;
+    else if (c === ">" || c === ")" || c === "]" || c === "}") depth = Math.max(0, depth - 1);
+    else if ((c === " " || c === "\t") && depth === 0) break;
+  }
+  return { type: rest.slice(0, i), rest: rest.slice(i).trim() };
+}
+
+/** A bare lowercase identifier — the shape druid uses for an optional `@return` name. */
+const RETURN_NAME = /^[a-z_][A-Za-z0-9_]*$/;
+
+function parseParam(rest: string): LibraryParam {
+  const spaceAt = rest.search(/\s/);
+  const rawName = spaceAt === -1 ? rest : rest.slice(0, spaceAt);
+  const afterName = spaceAt === -1 ? "" : rest.slice(spaceAt).trim();
+  const isVararg = rawName === "...";
+  const isOptional = !isVararg && rawName.endsWith("?");
+  const name = isOptional ? rawName.slice(0, -1) : rawName;
+  const { type, rest: doc } = readTypeToken(afterName);
+  return { name, types: type ? [type] : [], doc, isOptional, isVararg };
+}
+
+function parseReturn(rest: string): LibraryParam {
+  const { type, rest: afterType } = readTypeToken(rest);
+  const spaceAt = afterType.search(/\s/);
+  const head = spaceAt === -1 ? afterType : afterType.slice(0, spaceAt);
+  let name = "";
+  let doc = afterType;
+  if (head && RETURN_NAME.test(head)) {
+    name = head;
+    doc = spaceAt === -1 ? "" : afterType.slice(spaceAt).trim();
+  }
+  return { name, types: type ? [type] : [], doc, isOptional: false, isVararg: false };
+}
+
+const VISIBILITY_KEYWORDS = new Set<LibraryFieldVisibility>([
+  "public",
+  "protected",
+  "private",
+  "package",
+]);
+
+function parseField(rest: string): LibraryField {
+  // LuaLS grammar is `---@field [scope] <name> <type> [description]`. Strip a leading
+  // visibility keyword only when a further token follows it — a lone `---@field private`
+  // is a field literally named `private`, matching LuaLS's own resolution.
+  let body = rest;
+  let visibility: LibraryFieldVisibility | undefined;
+  const firstSpace = body.search(/\s/);
+  if (firstSpace !== -1) {
+    const first = body.slice(0, firstSpace);
+    if (VISIBILITY_KEYWORDS.has(first as LibraryFieldVisibility)) {
+      visibility = first as LibraryFieldVisibility;
+      body = body.slice(firstSpace).trim();
+    }
+  }
+  const spaceAt = body.search(/\s/);
+  const rawName = spaceAt === -1 ? body : body.slice(0, spaceAt);
+  const afterName = spaceAt === -1 ? "" : body.slice(spaceAt).trim();
+  const isOptional = rawName.endsWith("?");
+  const name = isOptional ? rawName.slice(0, -1) : rawName;
+  const { type, rest: doc } = readTypeToken(afterName);
+  return {
+    name,
+    types: type ? [type] : [],
+    doc,
+    isOptional,
+    ...(visibility ? { visibility } : {}),
+  };
+}
+
+function parseVararg(rest: string): LibraryParam {
+  const { type, rest: doc } = readTypeToken(rest);
+  return { name: "...", types: type ? [type] : [], doc, isOptional: false, isVararg: true };
+}
+
+function parseGenerics(rest: string): LibraryGeneric[] {
+  return rest
+    .split(",")
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0)
+    .map((part) => {
+      const colon = part.indexOf(":");
+      if (colon === -1) return { name: part.trim() };
+      return { name: part.slice(0, colon).trim(), constraint: part.slice(colon + 1).trim() };
+    });
+}
+
+/** Parse a `@class Name[ : parent]` head. The parent is kept as a single raw token. */
+function parseClassHead(rest: string): { name: string; extends?: string } {
+  const colon = rest.indexOf(":");
+  if (colon === -1) return { name: rest.trim() };
+  const parent = rest.slice(colon + 1).trim();
+  return { name: rest.slice(0, colon).trim(), ...(parent ? { extends: parent } : {}) };
+}
+
+interface FunctionDecl {
+  kind: "method" | "module";
+  receiver?: string;
+  name: string;
+}
+
+const FUNCTION_FORMS: { re: RegExp; kind: "method" | "module"; recv?: number; name: number }[] = [
+  { re: /^function\s+([A-Za-z_][\w.]*):([A-Za-z_]\w*)\s*\(/, kind: "method", recv: 1, name: 2 },
+  { re: /^function\s+([A-Za-z_][\w.]*)\.([A-Za-z_]\w*)\s*\(/, kind: "module", name: 2 },
+  { re: /^(?:local\s+)?function\s+([A-Za-z_]\w*)\s*\(/, kind: "module", name: 1 },
+  { re: /^([A-Za-z_][\w.]*):([A-Za-z_]\w*)\s*=\s*function\b/, kind: "method", recv: 1, name: 2 },
+  { re: /^([A-Za-z_][\w.]*)\.([A-Za-z_]\w*)\s*=\s*function\b/, kind: "module", name: 2 },
+  { re: /^([A-Za-z_]\w*)\s*=\s*function\b/, kind: "module", name: 1 },
+];
+
+function parseFunctionDecl(line: string): FunctionDecl | null {
+  for (const form of FUNCTION_FORMS) {
+    const m = form.re.exec(line);
+    if (!m) continue;
+    const name = m[form.name] ?? "";
+    if (form.kind === "method") {
+      const receiver = form.recv ? m[form.recv] : undefined;
+      return { kind: "method", name, ...(receiver ? { receiver } : {}) };
+    }
+    return { kind: "module", name };
+  }
+  return null;
+}
+
+const LOCAL_ASSIGN = /^local\s+([A-Za-z_]\w*)\s*=/;
+
+/**
+ * Scan one LuaLS-annotated source into a `LibraryModel`. Only column-0 lines are
+ * recognized (module- and class-level declarations and their leading `---@` block);
+ * indented lines — in-body closures, `---@cast`/`---@type` narrowing — are opaque,
+ * so they neither create declarations nor pollute the pending block. Output order
+ * follows source order, making the result stable across repeated runs.
+ */
+export function parseLualsSource(source: string): LibraryModel {
+  const interfaces: LibraryInterface[] = [];
+  const byName = new Map<string, LibraryInterface>();
+  const aliases: LibraryAlias[] = [];
+  const moduleFunctions: LibraryMethod[] = [];
+  const receiverBinding = new Map<string, string>();
+
+  let pending = emptyPending();
+  let openClass: LibraryInterface | null = null;
+  let lastOpenedClass: string | null = null;
+
+  const ensureInterface = (name: string): LibraryInterface => {
+    const existing = byName.get(name);
+    if (existing) return existing;
+    const created: LibraryInterface = {
+      name,
+      generics: [],
+      fields: [],
+      methods: [],
+      brief: "",
+    };
+    byName.set(name, created);
+    interfaces.push(created);
+    return created;
+  };
+
+  const methodFromPending = (name: string): LibraryMethod => ({
+    name,
+    brief: pending.doc.join("\n"),
+    generics: pending.generics,
+    params: pending.params,
+    returns: pending.returns,
+  });
+
+  for (const raw of source.split("\n")) {
+    // Column-0 discipline: a line with leading whitespace is opaque to the scanner.
+    if (/^\s/.test(raw) || raw.length === 0) continue;
+
+    if (raw.startsWith("---@")) {
+      const tagMatch = /^---@([a-zA-Z]+)\s*(.*)$/.exec(raw);
+      if (!tagMatch) continue;
+      const tag = tagMatch[1];
+      const rest = (tagMatch[2] ?? "").trim();
+      switch (tag) {
+        case "class": {
+          const head = parseClassHead(rest);
+          const iface = ensureInterface(head.name);
+          if (head.extends) iface.extends = head.extends;
+          if (pending.doc.length > 0 && iface.brief === "") iface.brief = pending.doc.join("\n");
+          if (pending.generics.length > 0) iface.generics = pending.generics;
+          openClass = iface;
+          lastOpenedClass = head.name;
+          pending = emptyPending();
+          break;
+        }
+        case "field": {
+          if (openClass) openClass.fields.push(parseField(rest));
+          break;
+        }
+        case "param": {
+          pending.params.push(parseParam(rest));
+          break;
+        }
+        case "vararg": {
+          pending.params.push(parseVararg(rest));
+          break;
+        }
+        case "return": {
+          pending.returns.push(parseReturn(rest));
+          break;
+        }
+        case "generic": {
+          pending.generics.push(...parseGenerics(rest));
+          break;
+        }
+        case "alias": {
+          const spaceAt = rest.search(/\s/);
+          const name = spaceAt === -1 ? rest : rest.slice(0, spaceAt);
+          const expr = spaceAt === -1 ? "" : rest.slice(spaceAt).trim();
+          aliases.push({ name, types: expr ? [expr] : [], doc: pending.doc.join("\n") });
+          pending = emptyPending();
+          break;
+        }
+        default:
+          // @private, @protected, @cast, @type, @diagnostic, @overload, ... — outside
+          // the Druid subset; recognized as a tag and skipped, never treated as doc.
+          break;
+      }
+      continue;
+    }
+
+    if (raw.startsWith("---")) {
+      pending.doc.push(raw.slice(3).trim());
+      continue;
+    }
+
+    const decl = parseFunctionDecl(raw);
+    if (decl) {
+      if (decl.kind === "method") {
+        const target = decl.receiver ? (receiverBinding.get(decl.receiver) ?? decl.receiver) : "";
+        ensureInterface(target).methods.push(methodFromPending(decl.name));
+      } else {
+        moduleFunctions.push(methodFromPending(decl.name));
+      }
+      pending = emptyPending();
+      openClass = null;
+      continue;
+    }
+
+    const localAssign = LOCAL_ASSIGN.exec(raw);
+    if (localAssign) {
+      const variable = localAssign[1];
+      if (variable && lastOpenedClass) receiverBinding.set(variable, lastOpenedClass);
+      lastOpenedClass = null;
+      openClass = null;
+      pending = emptyPending();
+    }
+  }
+
+  return { interfaces, aliases, moduleFunctions };
+}
+
+/**
+ * Fold several parsed models into one, merging interfaces by name (concatenating
+ * fields and methods, keeping the first non-empty `extends`/`brief`/`generics`) and
+ * concatenating aliases and module functions in argument order. Deterministic given
+ * a stable input order — the snapshot feeds it the fixture files sorted by path.
+ */
+export function mergeLibraryModels(models: LibraryModel[]): LibraryModel {
+  const interfaces: LibraryInterface[] = [];
+  const byName = new Map<string, LibraryInterface>();
+  const aliases: LibraryAlias[] = [];
+  const moduleFunctions: LibraryMethod[] = [];
+
+  for (const model of models) {
+    for (const iface of model.interfaces) {
+      const existing = byName.get(iface.name);
+      if (!existing) {
+        const copy: LibraryInterface = {
+          name: iface.name,
+          ...(iface.extends ? { extends: iface.extends } : {}),
+          generics: [...iface.generics],
+          fields: [...iface.fields],
+          methods: [...iface.methods],
+          brief: iface.brief,
+        };
+        byName.set(iface.name, copy);
+        interfaces.push(copy);
+        continue;
+      }
+      existing.fields.push(...iface.fields);
+      existing.methods.push(...iface.methods);
+      if (!existing.extends && iface.extends) existing.extends = iface.extends;
+      if (existing.brief === "" && iface.brief !== "") existing.brief = iface.brief;
+      if (existing.generics.length === 0 && iface.generics.length > 0) {
+        existing.generics = [...iface.generics];
+      }
+    }
+    aliases.push(...model.aliases);
+    moduleFunctions.push(...model.moduleFunctions);
+  }
+
+  return { interfaces, aliases, moduleFunctions };
+}

package/scripts/sync-luals-types.ts

@@ -0,0 +1,240 @@
+import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { buildFidelityReport, type FidelityReport } from "./luals-fidelity";
+import { mergeLibraryModels, parseLualsSource } from "./parse-luals";
+
+/**
+ * The LuaLS ingestion front-end pins its source per-entry: a druid-style library
+ * ships no `.d.ts`, only inline LuaLS `---@` annotations, and each such library
+ * lives in its own repo at its own tag. So every target carries its own
+ * `repo`/`ref`, unlike the ts-defold front-end's single shared `source`.
+ */
+export interface LualsTarget {
+  repo: string;
+  ref: string;
+  sourceGlobs: string[];
+  moduleId: string;
+  namespace: string;
+  typeRenames: Record<string, string>;
+  ignore: string[];
+}
+
+export interface LualsTargets {
+  targets: LualsTarget[];
+}
+
+const REQUIRED_FIELDS = ["repo", "ref", "sourceGlobs", "moduleId", "namespace"] as const;
+
+/**
+ * Read `luals-targets.json`, validate every required field per entry, and fill
+ * optional defaults (`typeRenames` → `{}`, `ignore` → `[]`). Throws on the first
+ * missing field naming both the field and the offending entry (its `moduleId`,
+ * or its index when `moduleId` itself is absent) — the loud-fail discipline the
+ * ts-defold `regenerate` uses for unmapped references. No network.
+ */
+export function readLualsTargets(packageRoot: string): LualsTarget[] {
+  const parsed = JSON.parse(readFileSync(join(packageRoot, "luals-targets.json"), "utf8")) as {
+    targets: Partial<LualsTarget>[];
+  };
+  return parsed.targets.map((entry, index) => {
+    const label = typeof entry.moduleId === "string" ? entry.moduleId : `index ${index}`;
+    for (const field of REQUIRED_FIELDS) {
+      if (entry[field] === undefined) {
+        throw new Error(`luals-targets.json: entry ${label} is missing required field "${field}".`);
+      }
+    }
+    return {
+      repo: entry.repo as string,
+      ref: entry.ref as string,
+      sourceGlobs: entry.sourceGlobs as string[],
+      moduleId: entry.moduleId as string,
+      namespace: entry.namespace as string,
+      typeRenames: entry.typeRenames ?? {},
+      ignore: entry.ignore ?? [],
+    };
+  });
+}
+
+/**
+ * Compile a glob to an anchored RegExp — mirrors `globToRegex` in
+ * `packages/cli/src/build-output.ts` rather than importing across packages
+ * (library-types must not depend on the cli package). A `**` path segment spans
+ * any number of segments, a bare `**` spans the rest, `*` a non-slash run, `?`
+ * one non-slash.
+ */
+function globToRegex(pattern: string): RegExp {
+  let out = "";
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === "*") {
+      if (pattern[i + 1] === "*") {
+        i++;
+        if (pattern[i + 1] === "/") {
+          i++;
+          out += "(?:[^/]+/)*";
+        } else {
+          out += ".*";
+        }
+      } else {
+        out += "[^/]*";
+      }
+    } else if (c === "?") {
+      out += "[^/]";
+    } else {
+      out += (c as string).replace(/[.+^$(){}|[\]\\]/g, "\\$&");
+    }
+  }
+  return new RegExp(`^${out}$`);
+}
+
+/**
+ * A path is selected iff at least one `sourceGlob` matches and no `ignore` glob
+ * matches. Returns the sorted, deduped subset — the fixture set the vendor step
+ * snapshots.
+ */
+export function selectLualsSources(
+  paths: string[],
+  target: { sourceGlobs: string[]; ignore: string[] },
+): string[] {
+  const includes = target.sourceGlobs.map(globToRegex);
+  const excludes = target.ignore.map(globToRegex);
+  const selected = paths.filter(
+    (p) => includes.some((re) => re.test(p)) && !excludes.some((re) => re.test(p)),
+  );
+  return [...new Set(selected)].sort();
+}
+
+/** Enumerate the pinned tree of a LuaLS library repo at its ref. Network seam. */
+export type ListLualsTree = (repo: string, ref: string) => Promise<string[]>;
+
+/** Fetch the raw text at a URL. Network seam — mirrors `sync-library-types.ts`. */
+export type FetchText = (url: string) => Promise<string>;
+
+/**
+ * A GitHub repo URL reduced to the bare `<owner>/<repo>` slug used to address
+ * raw content. Mirrors `repoSlug` in the ts-defold front-end.
+ */
+function repoSlug(repo: string): string {
+  return repo
+    .replace(/^https:\/\/github\.com\//, "")
+    .replace(/\.git$/, "")
+    .replace(/\/$/, "");
+}
+
+function rawUrl(target: LualsTarget, path: string): string {
+  return `https://raw.githubusercontent.com/${repoSlug(target.repo)}/${target.ref}/${path}`;
+}
+
+/**
+ * List the pinned tree, select the matching sources, fetch each via raw-content
+ * URL, and write it under `fixtures/luals/<namespace>/<relpath>` preserving tree
+ * shape. Snapshot only — no codemod. The `listTree`/`fetchText` seams keep the
+ * pass offline-testable; only the CLI `--fetch` arm wires the real network.
+ */
+export async function fetchLualsFixtures(
+  packageRoot: string,
+  target: LualsTarget,
+  seams: { listTree: ListLualsTree; fetchText: FetchText },
+): Promise<void> {
+  const paths = selectLualsSources(await seams.listTree(target.repo, target.ref), target);
+  for (const path of paths) {
+    const text = await seams.fetchText(rawUrl(target, path));
+    const dest = join(packageRoot, "fixtures/luals", target.namespace, path);
+    mkdirSync(dirname(dest), { recursive: true });
+    writeFileSync(dest, text);
+  }
+}
+
+/**
+ * Parse and merge a target's committed fixtures, then build its fidelity report.
+ * Reads only `fixtures/luals/<namespace>/**` from disk — zero network — so both
+ * the `--fidelity` CLI arm and its round-trip test drive the exact same path and
+ * agree byte-for-byte. Fixture files are read in sorted order for determinism,
+ * mirroring the parse snapshot.
+ */
+export function buildTargetFidelity(packageRoot: string, target: LualsTarget): FidelityReport {
+  const root = join(packageRoot, "fixtures/luals", target.namespace);
+  const files = readdirSync(root, { recursive: true })
+    .map((entry) => String(entry))
+    .filter((entry) => entry.endsWith(".lua"))
+    .sort();
+  const model = mergeLibraryModels(
+    files.map((rel) => parseLualsSource(readFileSync(join(root, rel), "utf8"))),
+  );
+  return buildFidelityReport(target.namespace, model, target.typeRenames);
+}
+
+/**
+ * A druid-style corpus member: a LuaLS-sourced pure-Lua library, distinct from
+ * the ts-defold hand-written modules. Standalone registry — the docs-site and
+ * CLI wirings belong to later slices, not this one.
+ */
+export interface LualsCorpusEntry {
+  moduleId: string;
+  namespace: string;
+  classification: "pure-lua";
+  source: "luals";
+}
+
+export function lualsCorpusTargets(packageRoot: string): LualsCorpusEntry[] {
+  return readLualsTargets(packageRoot).map((target) => ({
+    moduleId: target.moduleId,
+    namespace: target.namespace,
+    classification: "pure-lua",
+    source: "luals",
+  }));
+}
+
+interface GithubTreeResponse {
+  tree?: { path: string }[];
+}
+
+const githubHeaders = (): Record<string, string> => {
+  const token = process.env.GITHUB_TOKEN;
+  return token ? { Authorization: `Bearer ${token}` } : {};
+};
+
+const defaultListTree: ListLualsTree = async (repo, ref) => {
+  const url = `https://api.github.com/repos/${repoSlug(repo)}/git/trees/${ref}?recursive=1`;
+  const res = await fetch(url, { headers: githubHeaders() });
+  if (!res.ok) {
+    throw new Error(`git-trees fetch failed: ${url} -> ${res.status} ${res.statusText}`);
+  }
+  const body = (await res.json()) as GithubTreeResponse;
+  return (body.tree ?? []).map((e) => e.path);
+};
+
+const defaultFetchText: FetchText = async (url) => {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`fetch failed: ${url} -> ${res.status} ${res.statusText}`);
+  }
+  return res.text();
+};
+
+if (import.meta.main) {
+  const root = join(import.meta.dir, "..");
+  const argv = process.argv.slice(2);
+  if (argv.includes("--fetch")) {
+    const targets = readLualsTargets(root);
+    for (const target of targets) {
+      await fetchLualsFixtures(root, target, {
+        listTree: defaultListTree,
+        fetchText: defaultFetchText,
+      });
+      console.log(`snapshotted ${target.moduleId} from ${repoSlug(target.repo)}@${target.ref}`);
+    }
+  }
+  if (argv.includes("--fidelity")) {
+    const targets = readLualsTargets(root);
+    for (const target of targets) {
+      const report = buildTargetFidelity(root, target);
+      const dest = join(root, "fidelity", `${target.namespace}.json`);
+      mkdirSync(dirname(dest), { recursive: true });
+      writeFileSync(dest, `${JSON.stringify(report, null, 2)}\n`);
+      console.log(
+        `${target.moduleId}: coverage ${(report.coverage * 100).toFixed(1)}% (${report.unknownFallbacks} unknown, ${report.undocumentedMembers} undocumented)`,
+      );
+    }
+  }
+}