@defold-typescript/types 0.1.0

package/scripts/fidelity-audit.ts

@@ -0,0 +1,311 @@
+import { resolve } from "node:path";
+import { DEFOLD_TYPE_MAP } from "../src/core-types";
+import {
+  ARBITRARY_TABLE_SLOTS,
+  buildTableDocResolver,
+  HOMOGENEOUS_ARRAY_SLOTS,
+  MAPPING_TABLE_SLOTS,
+  parseTableFields,
+  recoverCallbackSignature,
+  TS_IDENTIFIER,
+} from "../src/emit-dts";
+import { parseMessagesDoc } from "../src/emit-messages";
+import {
+  collectConstantFqns,
+  generateModuleDeclaration,
+  MESSAGES_MANIFEST,
+  MODULE_MANIFEST,
+  type ModuleManifestEntry,
+} from "./regen";
+
+const NO_KNOWN_CONSTANTS: ReadonlySet<string> = new Set();
+
+// These message names are typed by builtin-messages-typing's separate surface
+// (BuiltinMessages), so a fixture `MESSAGE` element of the same name is not a
+// namespace-API loss — it is reclassified out of droppedElements below.
+const BUILTIN_MESSAGE_NAMES: ReadonlySet<string> = new Set(
+  parseMessagesDoc(MESSAGES_MANIFEST.doc).entries.map((e) => e.name),
+);
+
+export interface FidelityEntry {
+  droppedElements: number;
+  unknownTokens: string[];
+  recordTables: number;
+  multiReturn: number;
+  droppedMembers: number;
+  optionalAsRequired: number;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function elementsOf(doc: unknown): Record<string, unknown>[] {
+  if (!isRecord(doc) || !Array.isArray(doc.elements)) return [];
+  return doc.elements.filter(isRecord);
+}
+
+function stringArray(raw: unknown): string[] {
+  return Array.isArray(raw) ? raw.filter((item): item is string => typeof item === "string") : [];
+}
+
+function paramList(raw: unknown): Record<string, unknown>[] {
+  return Array.isArray(raw) ? raw.filter(isRecord) : [];
+}
+
+function docString(raw: unknown): string | undefined {
+  return typeof raw === "string" ? raw : undefined;
+}
+
+function isDocOptional(param: Record<string, unknown>): boolean {
+  return param.is_optional === "True" || stringArray(param.types).includes("nil");
+}
+
+function trailingOptionalCutoff(params: readonly Record<string, unknown>[]): number {
+  let cutoff = params.length;
+  for (let i = params.length - 1; i >= 0; i -= 1) {
+    const p = params[i];
+    if (p && isDocOptional(p)) cutoff = i;
+    else break;
+  }
+  return cutoff;
+}
+
+function auditEntry(
+  entry: ModuleManifestEntry,
+  knownConstantFqns: ReadonlySet<string>,
+): FidelityEntry {
+  const skipFunctions = new Set(entry.skipFunctions ?? []);
+  const elements = elementsOf(entry.doc);
+
+  let droppedElements = 0;
+  let recordTables = 0;
+  // multiReturn is fully recovered: emitReturn emits LuaMultiReturn<[...]> for
+  // every >1-return function, so no documented multi-return is a loss anymore.
+  const multiReturn = 0;
+  let optionalAsRequired = 0;
+  const unknown = new Set<string>();
+
+  const constantFqns = new Set<string>();
+  for (const element of elements) {
+    if (element.type === "CONSTANT" && typeof element.name === "string") {
+      constantFqns.add(element.name);
+    }
+  }
+
+  // Same module-scoped resolver the emitter builds, so a cross-reference table
+  // slot is recovered (not counted under recordTables) in lockstep with the
+  // emitted surface.
+  const resolver = buildTableDocResolver(
+    elements
+      .filter((element) => element.type === "FUNCTION" && typeof element.name === "string")
+      .map((element) => ({
+        name: element.name as string,
+        slots: [...paramList(element.parameters), ...paramList(element.returnvalues)].map(
+          (slot) => ({ types: stringArray(slot.types), doc: docString(slot.doc) ?? "" }),
+        ),
+      })),
+  );
+
+  const considerTypes = (
+    types: readonly string[],
+    doc?: string,
+    arbitraryTable = false,
+    mappingSlot?: { key: string; value: string },
+    homogeneousElement?: string | readonly string[],
+  ) => {
+    for (const token of types) {
+      // A `table` slot whose doc carries a parseable `<dl>` field list is
+      // recovered by emit-dts into an inline object type, so it no longer
+      // collapses to `Record`: don't count it under recordTables. Instead feed
+      // each recovered field's types back through considerTypes so an unmapped
+      // field token still surfaces under unknownTokens and a nested `table`
+      // field (no doc → not recovered) still counts under recordTables. This
+      // keeps the invariant that every loss in the emitted surface is measured.
+      if (token === "table") {
+        // A mapping-table slot is recovered by emit-dts into `LuaMap<K, V>` from
+        // the curated key/value tokens — not a `Record`, so don't count it.
+        // Feed the curated tokens back through considerTypes so an unmapped one
+        // still surfaces under unknownTokens (none today: hash/node/vector3 map).
+        if (mappingSlot !== undefined) {
+          considerTypes([mappingSlot.key, mappingSlot.value]);
+          continue;
+        }
+        // A homogeneous-array slot is recovered by emit-dts into `T[]` (or a
+        // `(A | B)[]` union element) from the curated element token(s) — not a
+        // `Record`, so don't count it. Feed the curated token(s) back through
+        // considerTypes so an unmapped one still surfaces under unknownTokens
+        // (none today: number/hash/string/url map).
+        if (homogeneousElement !== undefined) {
+          considerTypes(
+            typeof homogeneousElement === "string" ? [homogeneousElement] : homogeneousElement,
+          );
+          continue;
+        }
+        const fields = doc !== undefined ? parseTableFields(doc, resolver) : null;
+        if (fields !== null) {
+          // A recovered field carrying nested fields (the mixed `<dl>`+`<ul>`
+          // shape) emits a nested object, not a `Record` — recurse into the
+          // nested field types instead of counting the nested `table`.
+          for (const field of fields) {
+            if (field.fields !== undefined) {
+              for (const nested of field.fields) considerTypes(nested.types);
+            } else if (field.numberList === true) {
+              // Recovered as `number[]` by inlineTableType — no longer a
+              // `Record`, so skip it. Re-counting its `table` token here would
+              // double-count a slot the emitted surface no longer loses.
+            } else {
+              considerTypes(field.types);
+            }
+          }
+          continue;
+        }
+        // A slot on a serialization/JSON passthrough function is a genuinely
+        // arbitrary lua table — its emitted `Record<string | number, unknown>`
+        // is faithful, not a loss — so it is not counted under recordTables.
+        if (arbitraryTable) continue;
+        recordTables += 1;
+        continue;
+      }
+      // `nil` is the optional-parameter sentinel (emitParameter strips it), not a
+      // real Defold type the mapper fails to resolve — the optionalAsRequired
+      // category covers that loss instead. Constant FQNs defined in this module
+      // (constantFqns) or in any other manifest module (knownConstantFqns) now
+      // resolve to their brand type, and `function(...)` callback signatures
+      // recover to typed functions, so none is an unknown token.
+      if (
+        token !== "nil" &&
+        !Object.hasOwn(DEFOLD_TYPE_MAP, token) &&
+        !constantFqns.has(token) &&
+        !knownConstantFqns.has(token) &&
+        recoverCallbackSignature(token) === null
+      ) {
+        unknown.add(token);
+      }
+    }
+  };
+
+  for (const element of elements) {
+    const type = element.type;
+    // An identifier-named TYPEDEF is recovered into a per-namespace
+    // `type <name> = Opaque<"<name>">` alias, under the same TS_IDENTIFIER guard
+    // the emitter uses, so the gate and the emitted surface agree. A
+    // non-identifier TYPEDEF cannot become an alias and stays dropped.
+    if (
+      type === "TYPEDEF" &&
+      typeof element.name === "string" &&
+      TS_IDENTIFIER.test(element.name)
+    ) {
+      continue;
+    }
+    // A MESSAGE element whose name is in the builtin-messages catalog is a
+    // built-in message owned by builtin-messages-typing, not a namespace-API
+    // member — counting it here double-counts a surface that is fully typed
+    // elsewhere. A MESSAGE name absent from the catalog falls through to the
+    // catch-all and still counts, so the gate stays honest.
+    if (
+      type === "MESSAGE" &&
+      typeof element.name === "string" &&
+      BUILTIN_MESSAGE_NAMES.has(element.name)
+    ) {
+      continue;
+    }
+    if (type !== "FUNCTION" && type !== "VARIABLE" && type !== "CONSTANT" && type !== "PROPERTY") {
+      droppedElements += 1;
+      continue;
+    }
+    // PROPERTY is recovered into the per-namespace `interface properties` block;
+    // its name+type survive, so it is no longer a dropped element.
+    if (type === "CONSTANT" || type === "PROPERTY") continue;
+    if (type === "VARIABLE") {
+      considerTypes(stringArray(element.types));
+      continue;
+    }
+    // Skipped functions are measured as droppedMembers only; counting their
+    // params/returns here would double-count losses covered by hand-written d.ts.
+    if (typeof element.name === "string" && skipFunctions.has(stripNamespace(element.name))) {
+      continue;
+    }
+    const params = paramList(element.parameters);
+    const returns = paramList(element.returnvalues);
+    const cutoff = trailingOptionalCutoff(params);
+    const arbitraryTable =
+      typeof element.name === "string" && ARBITRARY_TABLE_SLOTS.has(element.name);
+    const mappingSlot =
+      typeof element.name === "string" ? MAPPING_TABLE_SLOTS.get(element.name) : undefined;
+    const homogeneousElement =
+      typeof element.name === "string" ? HOMOGENEOUS_ARRAY_SLOTS.get(element.name) : undefined;
+    params.forEach((param, index) => {
+      considerTypes(
+        stringArray(param.types),
+        docString(param.doc),
+        arbitraryTable,
+        mappingSlot,
+        homogeneousElement,
+      );
+      // Residual: a doc-optional param the emitter cannot mark `?` because a
+      // required param follows it. The trailing-run cutoff must match
+      // emit-dts so the gate and the emitted surface agree.
+      if (isDocOptional(param) && index < cutoff) optionalAsRequired += 1;
+    });
+    for (const ret of returns)
+      considerTypes(
+        stringArray(ret.types),
+        docString(ret.doc),
+        arbitraryTable,
+        mappingSlot,
+        homogeneousElement,
+      );
+  }
+
+  return {
+    droppedElements,
+    unknownTokens: [...unknown].sort(),
+    recordTables,
+    multiReturn,
+    droppedMembers: generateModuleDeclaration(entry, { knownConstantFqns: NO_KNOWN_CONSTANTS })
+      .dropped.length,
+    optionalAsRequired,
+  };
+}
+
+function stripNamespace(name: string): string {
+  const index = name.lastIndexOf(".");
+  return index === -1 ? name : name.slice(index + 1);
+}
+
+export function buildFidelityReport(
+  manifest: readonly ModuleManifestEntry[] = MODULE_MANIFEST,
+): Record<string, FidelityEntry> {
+  const report: Record<string, FidelityEntry> = {};
+  const knownConstantFqns = collectConstantFqns(manifest);
+  for (const namespace of [...manifest.map((e) => e.namespace)].sort()) {
+    const entry = manifest.find((e) => e.namespace === namespace);
+    if (entry) report[namespace] = auditEntry(entry, knownConstantFqns);
+  }
+  return report;
+}
+
+function biomeFormatJson(raw: string): string {
+  const out = Bun.spawnSync(
+    ["bunx", "biome", "format", "--stdin-file-path=fidelity-baseline.json"],
+    {
+      stdin: Buffer.from(raw),
+    },
+  );
+  if (out.exitCode !== 0) {
+    throw new Error(`biome format failed: ${out.stderr.toString()}`);
+  }
+  return out.stdout.toString();
+}
+
+if (import.meta.main) {
+  const report = buildFidelityReport();
+  if (process.argv.includes("--write")) {
+    const path = resolve(import.meta.dir, "fidelity-baseline.json");
+    Bun.write(path, biomeFormatJson(JSON.stringify(report)));
+    console.log(`wrote ${path}`);
+  } else {
+    console.log(JSON.stringify(report, null, 2));
+  }
+}

package/scripts/fidelity-baseline.json

@@ -0,0 +1,282 @@
+{
+  "b2d": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "buffer": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "camera": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "collectionfactory": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "collectionproxy": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "crash": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "factory": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "go": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 2,
+    "optionalAsRequired": 0
+  },
+  "graphics": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "gui": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "http": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "iac": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "iap": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 3,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "image": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "json": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "label": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "liveupdate": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "model": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "msg": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 1,
+    "optionalAsRequired": 0
+  },
+  "particlefx": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "physics": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 5,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "profiler": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "push": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 4,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "render": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 4,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "resource": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 3,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "socket": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 8,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "sound": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "sprite": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "sys": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "tilemap": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 2,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "timer": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "vmath": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "webview": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 1,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "window": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  },
+  "zlib": {
+    "droppedElements": 0,
+    "unknownTokens": [],
+    "recordTables": 0,
+    "multiReturn": 0,
+    "droppedMembers": 0,
+    "optionalAsRequired": 0
+  }
+}

package/scripts/materialize-version.ts

@@ -0,0 +1,51 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import {
+  type ApiTarget,
+  generateModuleDeclaration,
+  generateVersionIndex,
+  type ResolveTargetOptions,
+  resolveTargetModules,
+} from "./regen";
+
+export interface MaterializeVersionedSurfaceOptions {
+  readonly destDir: string;
+  readonly resolveOpts?: ResolveTargetOptions;
+  // Bare module names (no `.d.ts`) to omit from the surface — both the emitted
+  // file and its aggregate-index import. Lets a caller narrow the surface to a
+  // script kind without the generator knowing what a script kind is.
+  readonly excludeModules?: readonly string[];
+}
+
+// Generate a versioned surface on the fly into a project-local faux `@types`
+// package: resolve the target's module docs (ref-doc or committed fixture),
+// emit each module declaration, then write the aggregate side-effect entrypoint
+// and a minimal package.json. ref-doc targets are never pre-baked, so this is
+// the only path that turns a resolved version into a consumable type surface.
+export async function materializeVersionedSurface(
+  target: ApiTarget,
+  opts: MaterializeVersionedSurfaceOptions,
+): Promise<void> {
+  const exclude = new Set(opts.excludeModules ?? []);
+  const modules = (await resolveTargetModules(target, opts.resolveOpts ?? {})).filter(
+    (entry) => !exclude.has(entry.outFile.replace(/\.d\.ts$/, "")),
+  );
+  mkdirSync(opts.destDir, { recursive: true });
+
+  for (const entry of modules) {
+    const { contents } = generateModuleDeclaration(entry);
+    writeFileSync(resolve(opts.destDir, entry.outFile), contents);
+  }
+
+  const versioned = modules.map((entry) => ({ ...entry, versionId: target.id }));
+  writeFileSync(resolve(opts.destDir, "index.d.ts"), generateVersionIndex(target.id, versioned));
+
+  writeFileSync(
+    resolve(opts.destDir, "package.json"),
+    `${JSON.stringify(
+      { name: `@defold-typescript/materialized-${target.id}`, types: "index.d.ts" },
+      null,
+      2,
+    )}\n`,
+  );
+}