@defold-typescript/types 0.40.0 → 0.42.0

package/scripts/import-defold-release.ts

@@ -1,10 +1,12 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
-import { isKnownDefoldTypeToken } from "../src/emit-dts";
+import { unknownDefoldTypeLeaves } from "../src/emit-dts";
 import { refDocCacheDir, resolveRefDoc } from "./doc-source";
 import {
   apiElementIdentity,
   EDITOR_MANIFEST,
+  type ExtractedFixture,
+  extractAnnotationFixtures,
   IGNORED_UPSTREAM,
   mergeApiDocs,
   readZip,
@@ -13,7 +15,8 @@ import {
 } from "./sync-api-docs";
 
 const EXACT_VERSION = /^\d+\.\d+\.\d+$/;
-const LUA_NAMESPACE = /^[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)*$/;
+const LUA_NAMESPACE = /^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$/;
+const DECLARED_NAME_KINDS = new Set(["CONSTANT", "ENUM", "STRUCT", "TYPEDEF", "CLASS"]);
 const PACKAGE_ROOT = resolve(import.meta.dir, "..");
 
 // Editor-scripting namespaces are mapped, but not by `api-targets.json`: they
@@ -24,19 +27,48 @@ const PACKAGE_ROOT = resolve(import.meta.dir, "..");
 // excusing their bare names would suppress a real runtime namespace instead.
 const EDITOR_MAPPED_NAMESPACES = new Set(EDITOR_MANIFEST.map((source) => source.namespace));
 
-// The set of namespaces promoted into the generated surface at Defold 1.13.0.
-// Held here rather than derived from the root release model because the
+// Namespaces promoted into the generated surface for the first time at a given
+// release. Held here rather than derived from the root release model because the
 // per-package `rootDir` boundary forbids importing outside this package tree;
 // `scripts/release-model.test.ts` correspondence-guards this against the model.
-export const DEFOLD_1_13_PROMOTED_NAMESPACES = [
-  "b2d.chain",
-  "b2d.fixture",
-  "b2d.joint",
-  "b2d.shape",
-  "b2d.world",
-  "compute",
-  "material",
-] as const;
+export const PROMOTED_NAMESPACES_BY_VERSION: Readonly<Record<string, readonly string[]>> = {
+  "1.13.0": [
+    "b2d.chain",
+    "b2d.fixture",
+    "b2d.joint",
+    "b2d.shape",
+    "b2d.world",
+    "compute",
+    "material",
+  ],
+  "1.13.2": [
+    "bullet3d",
+    "bullet3d.collision_object",
+    "bullet3d.constraint",
+    "bullet3d.rigid_body",
+    "bullet3d.shape",
+    "bullet3d.world",
+  ],
+};
+
+function compareVersions(a: string, b: string): number {
+  const pa = a.split(".").map(Number);
+  const pb = b.split(".").map(Number);
+  for (let index = 0; index < Math.max(pa.length, pb.length); index += 1) {
+    const delta = (pa[index] ?? 0) - (pb[index] ?? 0);
+    if (delta !== 0) return delta;
+  }
+  return 0;
+}
+
+// A baseline that predates a promotion has no fixture for the promoted namespace,
+// so the import of the promoting release, and of every later one, seeds it empty.
+export function promotedNamespacesThrough(version: string): string[] {
+  return Object.keys(PROMOTED_NAMESPACES_BY_VERSION)
+    .filter((release) => compareVersions(release, version) <= 0)
+    .sort(compareVersions)
+    .flatMap((release) => PROMOTED_NAMESPACES_BY_VERSION[release] ?? []);
+}
 
 interface RawElement extends Record<string, unknown> {
   type?: string;
@@ -123,6 +155,9 @@ export interface ReleaseImportManifest {
     readonly unknownTypes: UnknownTypeBlocker[];
     readonly unmappedFunctionNamespaces: UnmappedNamespaceBlocker[];
   };
+  // The file names vendored under `annotations/`, which `loadAnnotations` holds
+  // the directory to. Empty before 1.13.2; absent from manifests written earlier.
+  readonly annotationFiles: readonly string[];
   readonly snapshots: Array<{
     readonly namespace: string;
     readonly fixture: string;
@@ -134,6 +169,8 @@ export interface ReleaseImportManifest {
 export type ReleaseImportPlan = Omit<ReleaseImportManifest, "snapshots"> & {
   readonly manifest: ReleaseImportManifest;
   readonly snapshots: ReleaseImportSnapshot[];
+  // The release's LuaLS annotation files, vendored verbatim. Empty before 1.13.2.
+  readonly annotations: ExtractedFixture[];
 };
 
 function isRecord(value: unknown): value is Record<string, unknown> {
@@ -159,12 +196,26 @@ function elementsOf(doc: unknown): RawElement[] {
   return doc.elements.filter(isRecord) as RawElement[];
 }
 
+// An ENUM's members are the constants 1.13.1 and earlier documented as
+// top-level CONSTANT elements, so each counts as a symbol beside the enum itself.
 function symbolNames(doc: unknown): string[] {
   return [
-    ...new Set(elementsOf(doc).flatMap((element) => (element.name ? [element.name] : []))),
+    ...new Set(
+      elementsOf(doc).flatMap((element) => [
+        ...(element.name ? [element.name] : []),
+        ...enumMemberNames(element),
+      ]),
+    ),
   ].sort();
 }
 
+function enumMemberNames(element: RawElement): string[] {
+  if (element.type !== "ENUM" || !Array.isArray(element.members)) return [];
+  return element.members.flatMap((member) =>
+    isRecord(member) && typeof member.name === "string" ? [member.name] : [],
+  );
+}
+
 function expectedSourceEntries(module: ReleaseBaselineModule): string[] {
   if (module.sourceEntries) return [...module.sourceEntries].sort();
   const mapped = SYNC_MANIFEST.find((entry) => entry.namespace === module.namespace);
@@ -180,29 +231,44 @@ function typeTokens(value: unknown, out: Set<string>): void {
   if (!isRecord(value)) return;
   if (Array.isArray(value.types)) {
     for (const token of value.types) {
-      if (typeof token === "string" && !isKnownDefoldTypeToken(token)) out.add(token);
+      if (typeof token === "string") addUnknownLeaves(token, out);
+    }
+  }
+  // A STRUCT member carries one `type` string rather than a `types[]` list; an
+  // ENUM member's is empty.
+  if (Array.isArray(value.members)) {
+    for (const member of value.members) {
+      if (isRecord(member) && typeof member.type === "string" && member.type !== "") {
+        addUnknownLeaves(member.type, out);
+      }
     }
   }
   for (const nested of Object.values(value)) typeTokens(nested, out);
 }
 
+function addUnknownLeaves(token: string, out: Set<string>): void {
+  for (const leaf of unknownDefoldTypeLeaves(token)) out.add(leaf);
+}
+
 function unknownTypeBlockers(
   namespace: string,
   doc: unknown,
-  knownConstants: ReadonlySet<string>,
+  knownNames: ReadonlySet<string>,
 ): UnknownTypeBlocker[] {
   const out: UnknownTypeBlocker[] = [];
   for (const element of elementsOf(doc)) {
     const tokens = new Set<string>();
     typeTokens(element, tokens);
-    for (const token of knownConstants) tokens.delete(token);
+    for (const token of tokens) if (knownNames.has(token)) tokens.delete(token);
     if (tokens.size === 0) continue;
     out.push({ namespace, symbol: element.name ?? "<unnamed>", tokens: [...tokens].sort() });
   }
   return out;
 }
 
-function reportWithoutSnapshots(plan: Omit<ReleaseImportPlan, "manifest">): ReleaseImportManifest {
+function reportWithoutSnapshots(
+  plan: Omit<ReleaseImportPlan, "manifest" | "annotations">,
+): ReleaseImportManifest {
   return {
     version: plan.version,
     baseline: plan.baseline,
@@ -212,6 +278,7 @@ function reportWithoutSnapshots(plan: Omit<ReleaseImportPlan, "manifest">): Rele
     moved: plan.moved,
     symbols: plan.symbols,
     blockers: plan.blockers,
+    annotationFiles: plan.annotationFiles,
     snapshots: plan.snapshots.map(({ namespace, fixture, sourceEntries, symbolSources }) => ({
       namespace,
       fixture,
@@ -264,13 +331,17 @@ export function buildReleaseImportPlan(input: {
     };
   });
 
-  const knownConstants = new Set<string>();
+  // Every constant and declared type name in any of the release's Lua docs, so a
+  // token naming one declared in another namespace, or only in a doc no namespace
+  // maps (`builtins`), is not an unknown type.
+  const knownNames = new Set<string>();
   for (const group of grouped.values()) {
     for (const { doc } of group) {
       for (const element of elementsOf(doc)) {
-        if (element.type === "CONSTANT" && typeof element.name === "string") {
-          knownConstants.add(element.name);
+        if (typeof element.name === "string" && DECLARED_NAME_KINDS.has(element.type ?? "")) {
+          knownNames.add(element.name);
         }
+        for (const member of enumMemberNames(element)) knownNames.add(member);
       }
     }
   }
@@ -325,7 +396,7 @@ export function buildReleaseImportPlan(input: {
     if (added.length > 0 || removed.length > 0)
       symbols.push({ namespace: module.namespace, added, removed });
     if (generatedNamespaces.has(module.namespace)) {
-      unknownTypes.push(...unknownTypeBlockers(module.namespace, doc, knownConstants));
+      unknownTypes.push(...unknownTypeBlockers(module.namespace, doc, knownNames));
     }
   }
 
@@ -356,6 +427,7 @@ export function buildReleaseImportPlan(input: {
     (a, b) => a.namespace.localeCompare(b.namespace) || a.symbol.localeCompare(b.symbol),
   );
   const ready = unknownTypes.length === 0 && unmappedFunctionNamespaces.length === 0;
+  const annotations = extractAnnotationFixtures(input.zip);
   const planBase = {
     version: input.version,
     baseline: input.baseline.id,
@@ -365,10 +437,11 @@ export function buildReleaseImportPlan(input: {
     moved,
     symbols,
     blockers: { unknownTypes, unmappedFunctionNamespaces },
+    annotationFiles: annotations.map((annotation) => annotation.namespace).sort(),
     snapshots,
   };
   const manifest = reportWithoutSnapshots(planBase);
-  return { ...planBase, manifest };
+  return { ...planBase, manifest, annotations };
 }
 
 export function releaseImportReportJson(plan: ReleaseImportPlan): string {
@@ -387,6 +460,13 @@ export function applyReleaseImport(plan: ReleaseImportPlan, packageRoot = PACKAG
     writeFileSync(path, `${JSON.stringify(snapshot.doc, null, 2)}\n`);
     written.push(relative);
   }
+  for (const annotation of plan.annotations) {
+    const relative = `${relativeRoot}/${annotation.fixture}`;
+    const path = resolve(packageRoot, relative);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, annotation.contents);
+    written.push(relative);
+  }
   // The prefixless builtins (`hash`, `pprint`, ...) are hand-vendored rather than
   // generated, so no upstream snapshot carries them. Seeding the canonical copy
   // here is what keeps a newly imported target from being born without a globals
@@ -442,7 +522,7 @@ export function parseReleaseImportArgs(args: readonly string[]): ReleaseImportAr
   return { version, check, json, ...(zipPath ? { zipPath } : {}) };
 }
 
-function loadBaseline(packageRoot = PACKAGE_ROOT): ReleaseBaseline {
+function loadBaseline(version: string, packageRoot = PACKAGE_ROOT): ReleaseBaseline {
   const registry = JSON.parse(readFileSync(resolve(packageRoot, "api-targets.json"), "utf8")) as {
     targets: Array<{
       id: string;
@@ -461,7 +541,7 @@ function loadBaseline(packageRoot = PACKAGE_ROOT): ReleaseBaseline {
   const modules = target.modules
     .filter((module) => SYNC_MANIFEST.some((source) => source.namespace === module.namespace))
     .map(load);
-  for (const namespace of DEFOLD_1_13_PROMOTED_NAMESPACES) {
+  for (const namespace of promotedNamespacesThrough(version)) {
     if (modules.some((module) => module.namespace === namespace)) continue;
     modules.push({
       namespace,
@@ -485,7 +565,7 @@ if (import.meta.main) {
     const plan = buildReleaseImportPlan({
       version: args.version,
       zip: resolved.zip,
-      baseline: loadBaseline(),
+      baseline: loadBaseline(args.version),
     });
     if (args.json) process.stdout.write(releaseImportReportJson(plan));
     else {

package/scripts/prose-constants.ts

@@ -1,8 +1,10 @@
 import type { ModuleManifestEntry } from "./regen";
 
-// A constant family the engine registers and one function's parameter prose
-// lists, but no ref-doc element declares. Each release's own list is the
-// evidence, so a surface declares exactly the members its release documents.
+// A constant family the engine registers and one function's prose names, but no
+// ref-doc element declares. Each release's own prose is the evidence, so a
+// surface declares exactly the members its release documents; the engine-binding
+// diff keeps every member registered. The prose is a parameter's `<li>` list, or,
+// with no `parameter`, a `<code>` mention in the function's description.
 interface ProseConstantFamily {
   readonly namespace: string;
   readonly prefix: string;
@@ -10,27 +12,55 @@ interface ProseConstantFamily {
   readonly evidence: {
     readonly namespace: string;
     readonly function: string;
-    readonly parameter: string;
+    readonly parameter?: string;
   };
 }
 
+const VERTEX_ATTRIBUTES = {
+  namespace: "material",
+  function: "material.set_vertex_attributes",
+  parameter: "attributes",
+} as const;
+
 const PROSE_CONSTANT_FAMILIES: readonly ProseConstantFamily[] = [
   {
     namespace: "graphics",
     prefix: "SEMANTIC_TYPE_",
     describes: "Vertex attribute semantic type",
-    evidence: {
-      namespace: "material",
-      function: "material.set_vertex_attributes",
-      parameter: "attributes",
-    },
+    evidence: VERTEX_ATTRIBUTES,
+  },
+  {
+    namespace: "graphics",
+    prefix: "DATA_TYPE_",
+    describes: "Vertex attribute data type",
+    evidence: VERTEX_ATTRIBUTES,
+  },
+  {
+    namespace: "graphics",
+    prefix: "COORDINATE_SPACE_",
+    describes: "Vertex attribute coordinate space",
+    evidence: VERTEX_ATTRIBUTES,
+  },
+  {
+    namespace: "material",
+    prefix: "CONSTANT_TYPE_",
+    describes: "Material constant type",
+    evidence: { namespace: "material", function: "material.set_constants", parameter: "constants" },
+  },
+  {
+    namespace: "render",
+    prefix: "TEXTURE_BIT",
+    describes: "Render target buffer flag that samples a depth or stencil buffer as a texture",
+    evidence: { namespace: "render", function: "render.render_target" },
   },
 ];
 
 interface DocElement {
   readonly type: string;
   readonly name: string;
+  readonly description?: string;
   readonly parameters?: readonly { readonly name: string; readonly doc?: string }[];
+  readonly members?: readonly { readonly name: string }[];
 }
 
 interface ApiDoc {
@@ -41,15 +71,36 @@ function escapeRegExp(text: string): string {
   return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
-function evidencedNames(family: ProseConstantFamily, modules: readonly ModuleManifestEntry[]) {
+function evidenceFunction(
+  family: ProseConstantFamily,
+  modules: readonly ModuleManifestEntry[],
+): DocElement | undefined {
   const source = modules.find((m) => m.namespace === family.evidence.namespace);
-  if (!source) return [];
-  const fn = (source.doc as ApiDoc).elements.find(
+  return (source?.doc as ApiDoc | undefined)?.elements.find(
     (e) => e.type === "FUNCTION" && e.name === family.evidence.function,
   );
-  const prose = fn?.parameters?.find((p) => p.name === family.evidence.parameter)?.doc ?? "";
+}
+
+// A release may declare the family itself, as `CONSTANT`s or `ENUM` members.
+function declaresFamily(family: ProseConstantFamily, modules: readonly ModuleManifestEntry[]) {
+  const target = modules.find((m) => m.namespace === family.namespace);
+  const prefix = `${family.namespace}.${family.prefix}`;
+  return ((target?.doc as ApiDoc | undefined)?.elements ?? []).some(
+    (e) =>
+      e.name.startsWith(prefix) ||
+      (e.members ?? []).some((member) => member.name.startsWith(prefix)),
+  );
+}
+
+function evidencedNames(family: ProseConstantFamily, fn: DocElement | undefined) {
+  const { parameter } = family.evidence;
+  const prose =
+    parameter === undefined
+      ? (fn?.description ?? "")
+      : (fn?.parameters?.find((p) => p.name === parameter)?.doc ?? "");
+  const name = escapeRegExp(`${family.namespace}.${family.prefix}`);
   const item = new RegExp(
-    `<li><code>(${escapeRegExp(`${family.namespace}.${family.prefix}`)}\\w+)</code></li>`,
+    parameter === undefined ? `<code>(${name}\\w*)</code>` : `<li><code>(${name}\\w+)</code></li>`,
     "g",
   );
   return [...new Set(Array.from(prose.matchAll(item), (m) => m[1] as string))];
@@ -78,8 +129,18 @@ export function synthesizeProseConstants(
 ): ModuleManifestEntry[] {
   let out = [...modules];
   for (const family of PROSE_CONSTANT_FAMILIES) {
-    const names = evidencedNames(family, out);
-    if (names.length === 0) continue;
+    const fn = evidenceFunction(family, out);
+    const names = evidencedNames(family, fn);
+    if (names.length === 0) {
+      // The evidence function survives but its prose lists no member: unless the
+      // release declares the family itself, the constants would vanish unannounced.
+      if (fn !== undefined && !declaresFamily(family, out)) {
+        throw new Error(
+          `prose constant family ${family.namespace}.${family.prefix}*: ${family.evidence.function} lists no member and ${family.namespace} declares none`,
+        );
+      }
+      continue;
+    }
     const index = out.findIndex((m) => m.namespace === family.namespace);
     const target = out[index];
     if (!target) continue;