@defold-typescript/library-types 0.22.0 → 0.24.0

package/scripts/sync-authored-types.ts

@@ -0,0 +1,249 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { extractApiDoc } from "./extract-api-doc";
+import { type ApiDocElement, importUpstreamDocs } from "./import-upstream-docs";
+import { type LuaMember, parseLuaSurface } from "./parse-lua-surface";
+
+/**
+ * The authored/forked front-end: a fourth `library-types` corpus mode beside the
+ * LuaLS (`sync-luals-types.ts`), `.script_api` (`sync-script-api-types.ts`), and
+ * markdown (`sync-markdown-types.ts`) lanes. The other three ingest a *non-*`.d.ts`
+ * primary source and generate a declaration; this one is for a library whose
+ * upstream has no usable structured source, so its `.d.ts` is hand-authored or
+ * forked from the retired ts-defold `generated/` golden and vendored under
+ * `fixtures/authored/<moduleId>.d.ts`.
+ *
+ * A vendored authored `.d.ts` is already target-form — a `declare module
+ * '<moduleId>'` ambient — so the lane is light: it emits the vendored source
+ * verbatim as the `generated/<namespace>.d.ts` golden and runs
+ * `extractApiDoc` for the `api-doc/<namespace>.json` the docs-site consumes. The
+ * api-doc alone carries one thing the fork does not: where a target vendors its
+ * upstream `.lua` and the fork documents a member nowhere, upstream's own LuaDoc
+ * summary is lowered in, marked `docSource: "upstream"` (see
+ * `import-upstream-docs.ts`). The *emitted* surface is still the vendored authored
+ * source untouched, so the go/no-go gate is a
+ * forked-vs-generated identity diff (see the golden loop) — emission fidelity.
+ * There is no *type* coverage comparison because upstream declares no types; the
+ * *surface* is another matter, and a target that vendors its upstream `.lua` under
+ * `upstreamLua` is measured on member names and arity by `authored-parity.ts`.
+ * Beyond that, a corrected fork's accuracy rests on its manual audit against
+ * upstream plus the per-library shape assertions in the sibling test.
+ */
+/**
+ * Why a target's surface cannot be measured against upstream. A closed set: the
+ * alternative to `upstreamLua` has to be a *chosen* category, because free prose
+ * would let "nobody looked yet" and "looked, and it is genuinely unmeasurable"
+ * read the same in the config.
+ *
+ * - `unresolved-path` — the `moduleId`-derived path does not resolve at the
+ *   pinned `ref` (including a repository that no longer exists there).
+ * - `no-module-file` — the upstream resolves but declares no single module file
+ *   this target could be measured against.
+ * - `unparseable-shape` — the module file exists but closes in a way
+ *   `parseLuaSurface` refuses, such as a metatable that could delegate members
+ *   no column-0 scan would find.
+ */
+export const PARITY_VERDICT_REASONS = [
+  "no-module-file",
+  "unresolved-path",
+  "unparseable-shape",
+] as const;
+
+export type ParityVerdictReason = (typeof PARITY_VERDICT_REASONS)[number];
+
+export interface ParityVerdict {
+  reason: ParityVerdictReason;
+  note: string;
+}
+
+export interface AuthoredTarget {
+  repo: string;
+  ref: string;
+  // Package-relative path of the vendored authored/forked `.d.ts`
+  // (`fixtures/authored/<moduleId>.d.ts`).
+  authored: string;
+  moduleId: string;
+  namespace: string;
+  generated: string;
+  apiDoc: string;
+  // Defaults to `fidelity/<namespace>.json` when omitted. A fork carries no
+  // fidelity artifact — there is no primary source to measure against, and the
+  // emit is lossless by construction — but the field mirrors the sibling lanes
+  // so a future hand-authored target can record one.
+  fidelity: string;
+  // SPDX-style license id, surfaced by the docs-site provenance block. Optional
+  // in the config; defaults to "".
+  license: string;
+  // Package-relative paths of the vendored upstream `.lua` sources this target's
+  // surface is measured against (`authored-parity.ts`). Optional and defaulted to
+  // `[]`: a target with no vendored upstream is simply unmeasured, which is why it
+  // must not join `REQUIRED_FIELDS`.
+  upstreamLua: string[];
+  // Why this target vendors no upstream `.lua`. Mutually exclusive with a
+  // non-empty `upstreamLua`, and one of the two is mandatory: neither field is
+  // required on its own, but the *disjunction* is, so a target cannot land
+  // unexamined and read as measured-and-clean. Absent on a measured target.
+  parityVerdict?: ParityVerdict;
+}
+
+export interface AuthoredTargets {
+  targets: AuthoredTarget[];
+}
+
+const REQUIRED_FIELDS = [
+  "repo",
+  "ref",
+  "authored",
+  "moduleId",
+  "namespace",
+  "generated",
+  "apiDoc",
+] as const;
+
+/** Validate the parity declaration: exactly one of a non-empty `upstreamLua` and
+ * a well-formed `parityVerdict`. Enforced here rather than through
+ * `REQUIRED_FIELDS` because neither field is required on its own — the bijection
+ * between them is the gate. */
+function validateParityDeclaration(
+  label: string,
+  upstreamLua: string[],
+  verdict: ParityVerdict | undefined,
+): void {
+  if (upstreamLua.length > 0 && verdict !== undefined) {
+    throw new Error(
+      `authored-targets.json: entry ${label} declares both upstreamLua and parityVerdict — a measured target carries no verdict.`,
+    );
+  }
+  if (upstreamLua.length === 0 && verdict === undefined) {
+    throw new Error(
+      `authored-targets.json: entry ${label} declares neither upstreamLua nor parityVerdict — vendor its upstream .lua, or record a parityVerdict saying why it cannot be measured.`,
+    );
+  }
+  if (verdict === undefined) return;
+  if (!(PARITY_VERDICT_REASONS as readonly string[]).includes(verdict.reason)) {
+    throw new Error(
+      `authored-targets.json: entry ${label} has parityVerdict reason "${verdict.reason}", which is not one of ${PARITY_VERDICT_REASONS.join(", ")}.`,
+    );
+  }
+  if (typeof verdict.note !== "string" || verdict.note.trim() === "") {
+    throw new Error(
+      `authored-targets.json: entry ${label} has a parityVerdict with an empty note — say what was looked at and why it could not be measured.`,
+    );
+  }
+}
+
+/**
+ * Read `authored-targets.json`, validate every required field per entry, and fill
+ * optional defaults (`fidelity` → `fidelity/<namespace>.json`, `license` → "",
+ * `upstreamLua` → []).
+ * Throws on the first missing field naming both the field and the offending entry
+ * (its `moduleId`, or its index when `moduleId` itself is absent) — the loud-fail
+ * discipline `readMarkdownTargets`/`readScriptApiTargets`/`readLualsTargets` use.
+ * Also throws when an entry's parity declaration is missing, doubled, or
+ * malformed (see `validateParityDeclaration`).
+ */
+export function readAuthoredTargets(packageRoot: string): AuthoredTarget[] {
+  const parsed = JSON.parse(readFileSync(join(packageRoot, "authored-targets.json"), "utf8")) as {
+    targets: Partial<AuthoredTarget>[];
+  };
+  return parsed.targets.map((entry, index) => {
+    const label = typeof entry.moduleId === "string" ? entry.moduleId : `index ${index}`;
+    for (const field of REQUIRED_FIELDS) {
+      if (entry[field] === undefined) {
+        throw new Error(
+          `authored-targets.json: entry ${label} is missing required field "${field}".`,
+        );
+      }
+    }
+    const upstreamLua = entry.upstreamLua ?? [];
+    validateParityDeclaration(label, upstreamLua, entry.parityVerdict);
+    return {
+      repo: entry.repo as string,
+      ref: entry.ref as string,
+      authored: entry.authored as string,
+      moduleId: entry.moduleId as string,
+      namespace: entry.namespace as string,
+      generated: entry.generated as string,
+      apiDoc: entry.apiDoc as string,
+      fidelity: entry.fidelity ?? `fidelity/${entry.namespace as string}.json`,
+      license: entry.license ?? "",
+      upstreamLua,
+      ...(entry.parityVerdict === undefined ? {} : { parityVerdict: entry.parityVerdict }),
+    };
+  });
+}
+
+function readAuthoredSource(packageRoot: string, target: AuthoredTarget): string {
+  return readFileSync(join(packageRoot, target.authored), "utf8");
+}
+
+/**
+ * The `generated/<namespace>.d.ts` golden, under whatever stem the entry pins —
+ * a bare segment or the dotted `moduleId`. The vendored authored
+ * `.d.ts` is already a `declare module '<moduleId>'` ambient, so the emit is the
+ * source verbatim — the vendored surface passes through unchanged, which is what
+ * makes the emission lossless. It says nothing about how that vendored surface
+ * compares to upstream.
+ */
+export function emitAuthoredDeclaration(packageRoot: string, target: AuthoredTarget): string {
+  return readAuthoredSource(packageRoot, target);
+}
+
+/** The pinned upstream members by name, the way `authored-parity.ts` builds them:
+ * a later definition of a name wins, matching Lua. Empty for a target that vendors
+ * no upstream `.lua`, which is then never parsed. */
+function upstreamMembers(packageRoot: string, target: AuthoredTarget): Map<string, LuaMember> {
+  const members = new Map<string, LuaMember>();
+  for (const relative of target.upstreamLua) {
+    for (const member of parseLuaSurface(readFileSync(join(packageRoot, relative), "utf8")).members)
+      members.set(member.name, member);
+  }
+  return members;
+}
+
+/**
+ * `extractApiDoc` the vendored `.d.ts` under the pinned publish namespace, plus
+ * upstream's own LuaDoc for the members the fork documents nowhere, pretty-printed
+ * as the `api-doc/<namespace>.json` golden.
+ *
+ * The api-doc is therefore the fork *plus* imported upstream prose, and it is the
+ * only artifact that is: the emitted `generated/<namespace>.d.ts` stays the vendored
+ * source verbatim, so the identity diff still proves the emit is lossless. An
+ * imported element carries `docSource: "upstream"` through to the docs-site, which
+ * marks it rather than presenting it as first-party. See `import-upstream-docs.ts`
+ * for the merge rules.
+ */
+export function lowerAuthoredApiDoc(packageRoot: string, target: AuthoredTarget): string {
+  // `extractApiDoc` returns `unknown`; the merge needs the element list, and
+  // overwriting the existing `elements` key by spread keeps it in place, so the
+  // golden's key order is unchanged.
+  const doc = extractApiDoc(readAuthoredSource(packageRoot, target), target.namespace) as Record<
+    string,
+    unknown
+  > & { elements: ApiDocElement[] };
+  const elements = importUpstreamDocs(doc.elements, upstreamMembers(packageRoot, target));
+  return `${JSON.stringify({ ...doc, elements }, null, 2)}\n`;
+}
+
+if (import.meta.main) {
+  const root = join(import.meta.dir, "..");
+  const argv = process.argv.slice(2);
+  if (argv.includes("--emit")) {
+    for (const target of readAuthoredTargets(root)) {
+      const contents = emitAuthoredDeclaration(root, target);
+      const dest = join(root, target.generated);
+      mkdirSync(dirname(dest), { recursive: true });
+      writeFileSync(dest, contents);
+      console.log(`emitted ${target.moduleId} -> ${target.generated}`);
+    }
+  }
+  if (argv.includes("--api-doc")) {
+    for (const target of readAuthoredTargets(root)) {
+      const json = lowerAuthoredApiDoc(root, target);
+      const dest = join(root, target.apiDoc);
+      mkdirSync(dirname(dest), { recursive: true });
+      writeFileSync(dest, json);
+      console.log(`lowered ${target.moduleId} -> ${target.apiDoc}`);
+    }
+  }
+}

package/scripts/sync-library-types.ts

@@ -2,6 +2,11 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import ts from "typescript";
 import { extractApiDoc } from "./extract-api-doc";
+import { readAuthoredTargets } from "./sync-authored-types";
+import { readLualsTargets } from "./sync-luals-types";
+import { readMarkdownTargets } from "./sync-markdown-types";
+import { readOpenApiTargets } from "./sync-openapi-types";
+import { readScriptApiTargets } from "./sync-script-api-types";
 
 /**
  * ts-defold/library core-type references -> the @defold-typescript/types surface.
@@ -330,6 +335,65 @@ export function classifyLibraryDirs(
     .sort((a, b) => (a.dir < b.dir ? -1 : a.dir > b.dir ? 1 : 0));
 }
 
+/**
+ * The union every maintained-here lane contributes: authored `moduleId`s, and
+ * the `namespace`s of the LuaLS, script_api, OpenAPI and markdown front-ends.
+ * Namespaces are matched in both forms because the lanes use both — druid-style
+ * libraries register a bare `tweener`, while `defold-saver` registers the dotted
+ * `saver.saver` / `saver.storage`.
+ */
+export interface MaintainedHereRegistry {
+  moduleIds: ReadonlySet<string>;
+  namespaces: ReadonlySet<string>;
+}
+
+/**
+ * The subset of `modules` this repo maintains itself rather than sourcing from
+ * ts-defold/library. A live `library-targets.json` row always wins, so a module
+ * registered on a maintained-here lane *and* still vendored stays
+ * ts-defold-sourced and keeps its dir classified `already-vendored`. The
+ * registry is empty today, so no committed module takes that branch; it is what
+ * a returning row would take.
+ */
+export function maintainedHereModules(
+  modules: readonly string[],
+  registry: MaintainedHereRegistry,
+  liveModules: ReadonlySet<string>,
+): Set<string> {
+  return new Set(
+    modules.filter(
+      (m) =>
+        !liveModules.has(m) &&
+        (registry.moduleIds.has(m) ||
+          registry.namespaces.has(m) ||
+          registry.namespaces.has(m.split(".")[0] as string)),
+    ),
+  );
+}
+
+/**
+ * The upstream dirs this repo has severed onto a maintained-here lane. Severance
+ * deletes a library's `library-targets.json` row, but its dir still exists
+ * upstream — so a regen would re-add it under whatever its module shape implies
+ * unless it is filtered out first. The registries give an exact derivation with
+ * no hand-maintained list: a dir is severed iff it has at least one module and
+ * every module is maintained here. A dir mixing maintained-here and live modules
+ * is still upstream-sourced and stays classified — `nakama-defold` was the last
+ * such dir, and severed whole once its live row went.
+ */
+export function severedDirsFromModules(
+  modulesByDir: Map<string, string[]>,
+  registry: MaintainedHereRegistry,
+  liveModules: ReadonlySet<string>,
+): Set<string> {
+  const severed = new Set<string>();
+  for (const [dir, modules] of modulesByDir) {
+    const maintained = maintainedHereModules(modules, registry, liveModules);
+    if (modules.length > 0 && maintained.size === modules.length) severed.add(dir);
+  }
+  return severed;
+}
+
 interface GithubTreeResponse {
   tree?: { path: string }[];
 }
@@ -376,8 +440,28 @@ export type ListTree = (source: LibrarySource) => Promise<string[]>;
 const defaultListTree: ListTree = (source) => githubTreePaths(repoSlug(source.repo), source.commit);
 
 /**
- * Enumerate every ts-defold/library dir at the pin, classify each by its
- * module-name shape, and write `library-classification.json`. The `listTree`
+ * Read every maintained-here registry off the package root and union them.
+ * Each reader `readFileSync`s unconditionally and throws on a missing required
+ * field — deliberately no absent-file fallback, so a lane added without its
+ * fixture fails loudly rather than silently un-severing that lane's dirs.
+ */
+export function readMaintainedHereRegistry(packageRoot: string): MaintainedHereRegistry {
+  return {
+    moduleIds: new Set(readAuthoredTargets(packageRoot).map((t) => t.moduleId)),
+    namespaces: new Set([
+      ...readLualsTargets(packageRoot).map((t) => t.namespace),
+      ...readScriptApiTargets(packageRoot).map((t) => t.namespace),
+      ...readOpenApiTargets(packageRoot).map((t) => t.namespace),
+      ...readMarkdownTargets(packageRoot).map((t) => t.namespace),
+    ]),
+  };
+}
+
+/**
+ * Enumerate every ts-defold/library dir at the pin, drop the dirs severed onto
+ * a maintained-here lane, classify each survivor by its module-name shape, and write
+ * `library-classification.json`. Filtering happens here, not in
+ * `classifyLibraryDirs`, so that pass stays a pure map-and-sort. The `listTree`
  * seam keeps the pass offline-testable; only the CLI wires the real call, and it
  * stays out of CI (mirrors `--check`). The manifest pins the same `source` as
  * `library-targets.json`.
@@ -392,9 +476,21 @@ export async function writeClassification(
     targets.map((t) => t.path.split("/")[1]).filter((d): d is string => d !== undefined),
   );
   const coveredByGoalDirs = new Set(["defold-lldebugger", "defold-xmath"]);
+  const liveModules = new Set(targets.map((t) => t.module));
+
+  const registry = readMaintainedHereRegistry(packageRoot);
 
   const modulesByDir = libraryModulesFromTree(await listTree(source));
-  const dirs = [...modulesByDir].map(([dir, modules]) => ({ dir, modules }));
+  const severedDirs = severedDirsFromModules(modulesByDir, registry, liveModules);
+  const dirs = [...modulesByDir]
+    // Vendored wins: a stale maintained-here entry can never erase a live ts-defold row.
+    .filter(([dir]) => !severedDirs.has(dir) || vendoredDirs.has(dir))
+    // A survivor records only its ts-defold-sourced remainder as evidence, so its
+    // written shape agrees with its own `modules` array.
+    .map(([dir, modules]) => {
+      const maintained = maintainedHereModules(modules, registry, liveModules);
+      return { dir, modules: modules.filter((m) => !maintained.has(m)) };
+    });
   const entries = classifyLibraryDirs(dirs, { vendoredDirs, coveredByGoalDirs });
   writeFileSync(
     join(packageRoot, "library-classification.json"),

package/scripts/sync-luals-types.ts

@@ -1,6 +1,7 @@
 import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { emitLibraryDeclarations } from "./emit-library-dts";
+import { type AnnotationOverrides, applyAnnotationOverrides } from "./apply-luals-overrides";
+import { type ExternalTypeRef, emitLibraryDeclarations } from "./emit-library-dts";
 import { lowerLibraryModel } from "./lower-api-doc";
 import { buildFidelityReport, type FidelityReport } from "./luals-fidelity";
 import { type LibraryModel, mergeLibraryModels, parseLualsSource } from "./parse-luals";
@@ -17,11 +18,24 @@ export interface LualsTarget {
   sourceGlobs: string[];
   moduleId: string;
   namespace: string;
+  // Token -> TypeScript type *text*, not an identifier: the mapper emits the value
+  // verbatim. The value may name an ambient global that needs no import — a core
+  // rename (`Vector3`, `Opaque<"node">`) or a type the Lua stdlib already declares
+  // (`debug.FunctionInfo`) — which is what distinguishes it from `externalTypes`.
   typeRenames: Record<string, string>;
   ignore: string[];
   // SPDX-style license id, surfaced by the docs-site provenance block. Optional
   // in the config; the docs-site defaults an absent value to "".
   license?: string;
+  // Post-merge corrections applied in `buildTargetModel` for the cases where an
+  // upstream annotation the fixtures freeze diverges from the runtime. Optional and
+  // loud-failing on an absent key — see `apply-luals-overrides.ts`.
+  annotationOverrides?: AnnotationOverrides;
+  // Type tokens this library references but another target owns, keyed by the token:
+  // the emitter writes a real cross-module `import` for each. A project consuming this
+  // library must declare the named upstream dependency too — `resolve` materializes no
+  // inter-library graph. See `emit-library-dts.ts`.
+  externalTypes?: Record<string, ExternalTypeRef>;
 }
 
 export interface LualsTargets {
@@ -57,6 +71,10 @@ export function readLualsTargets(packageRoot: string): LualsTarget[] {
       typeRenames: entry.typeRenames ?? {},
       ignore: entry.ignore ?? [],
       ...(entry.license !== undefined ? { license: entry.license } : {}),
+      ...(entry.annotationOverrides !== undefined
+        ? { annotationOverrides: entry.annotationOverrides }
+        : {}),
+      ...(entry.externalTypes !== undefined ? { externalTypes: entry.externalTypes } : {}),
     };
   });
 }
@@ -188,7 +206,13 @@ export function buildTargetModel(
   const parsed = new Map<string, LibraryModel>();
   for (const rel of files) parsed.set(rel, parseLualsSource(readFileSync(join(root, rel), "utf8")));
   const merged = mergeLibraryModels([...parsed.values()]);
-  return { ...merged, moduleFunctions: parsed.get(ownFile)?.moduleFunctions ?? [] };
+  const model: LibraryModel = {
+    ...merged,
+    moduleFunctions: parsed.get(ownFile)?.moduleFunctions ?? [],
+  };
+  return target.annotationOverrides
+    ? applyAnnotationOverrides(model, target.annotationOverrides)
+    : model;
 }
 
 export function buildTargetFidelity(packageRoot: string, target: LualsTarget): FidelityReport {
@@ -196,6 +220,7 @@ export function buildTargetFidelity(packageRoot: string, target: LualsTarget): F
     target.namespace,
     buildTargetModel(packageRoot, target),
     target.typeRenames,
+    target.externalTypes,
   );
 }
 
@@ -279,6 +304,7 @@ if (import.meta.main) {
       const declarations = emitLibraryDeclarations(model, {
         moduleId: target.moduleId,
         typeRenames: target.typeRenames,
+        externalTypes: target.externalTypes,
       });
       const dest = join(root, "generated", `${target.namespace}.d.ts`);
       mkdirSync(dirname(dest), { recursive: true });
@@ -293,6 +319,7 @@ if (import.meta.main) {
       const lowered = lowerLibraryModel(model, {
         namespace: target.namespace,
         typeRenames: target.typeRenames,
+        externalTypes: target.externalTypes,
       });
       const dest = join(root, "api-doc", `${target.namespace}.json`);
       mkdirSync(dirname(dest), { recursive: true });