@defold-typescript/transpiler 0.42.0 → 0.44.0

package/dist/transpile.d.ts

@@ -21,6 +21,7 @@ export interface TranspileProjectResult {
     readonly companions?: Readonly<Record<string, string>>;
     readonly lualib?: string;
     readonly timersRuntime?: string;
+    readonly hsmModules?: Readonly<Record<string, string>>;
 }
 export declare const AMBIENT_FILES: Readonly<Record<string, string>>;
 interface CollectableFile {
@@ -31,5 +32,15 @@ interface CollectableFile {
 }
 export declare function collectOutputs(transpiledFiles: readonly CollectableFile[], diagnostics: readonly ts.Diagnostic[], userKeys: ReadonlySet<string>, emittedCompanions?: ReadonlyMap<string, string>): TranspileProjectResult;
 export declare function transpileProject(input: TranspileProjectInput): TranspileProjectResult;
+/**
+ * Every `hsm` module compiled to Lua, keyed by module name, once per process.
+ * The source is the shipped TypeScript, never a prebuilt Lua copy, so the Lua a
+ * project gets always comes from the compiler that builds it. Each file sits at
+ * `defold_typescript_hsm/<module>.ts`, so a require between modules is already
+ * spelled the way the user's lowered imports are. Throws when the compile
+ * reports anything, or when a module outside `HSM_LUALIB_MODULES` needs the
+ * lualib bundle.
+ */
+export declare function compileHsmModules(): Readonly<Record<string, string>>;
 export declare function transpile(source: string): TranspileResult;
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defold-typescript/transpiler",
-  "version": "0.42.0",
+  "version": "0.44.0",
   "description": "TypeScript-to-Lua build pipeline tuned for Defold's runtime.",
   "license": "MIT",
   "repository": {
@@ -30,13 +30,14 @@
     "access": "public"
   },
   "scripts": {
-    "build": "rm -rf dist && bun build src/index.ts --target=node --format=esm --packages=external --outdir=dist && tsc -p tsconfig.build.json --emitDeclarationOnly",
+    "build": "rm -rf dist && bun scripts/copy-hsm.ts && bun build src/index.ts --target=node --format=esm --packages=external --outdir=dist && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test": "bun test"
   },
   "dependencies": {
-    "@defold-typescript/types": "0.42.0",
+    "@defold-typescript/types": "0.44.0",
     "@typescript-to-lua/language-extensions": "1.19.0",
+    "typescript": "6.0.2",
     "typescript-to-lua": "^1.36.0"
   }
 }

package/src/emitted-requires.ts

@@ -1,10 +1,12 @@
+import { isHsmRequireName } from "./hsm-builtin";
 import { scanEmittedRequires } from "./lua-require-scan";
 import { TIMERS_REQUIRE_NAME } from "./timers-runtime";
 
 export const LUALIB_REQUIRE_NAME = "lualib_bundle";
 
-// Written unconditionally by both build paths and backed by no TypeScript
-// source, so a require of either can never be unresolvable.
+// Written by both build paths whenever a chunk requires them and backed by no
+// TypeScript source, so a require of one can never be unresolvable. The hsm
+// modules join them through `isHsmRequireName`.
 const GENERATED_RUNTIME_REQUIRES: ReadonlySet<string> = new Set([
   LUALIB_REQUIRE_NAME,
   TIMERS_REQUIRE_NAME,
@@ -12,7 +14,7 @@ const GENERATED_RUNTIME_REQUIRES: ReadonlySet<string> = new Set([
 
 /**
  * The distinct string-literal module paths an emitted Lua chunk requires, in
- * source order. The two generated runtimes are skipped, as is any require whose
+ * source order. The generated runtimes and hsm modules are skipped, as is any require whose
  * argument is not a string literal — neither can name a source the build owns.
  */
 export function findEmittedRequires(lua: string): string[] {
@@ -20,7 +22,7 @@ export function findEmittedRequires(lua: string): string[] {
   const seen = new Set<string>();
 
   for (const { path } of scanEmittedRequires(lua)) {
-    if (GENERATED_RUNTIME_REQUIRES.has(path) || seen.has(path)) {
+    if (GENERATED_RUNTIME_REQUIRES.has(path) || isHsmRequireName(path) || seen.has(path)) {
       continue;
     }
     seen.add(path);

package/src/hsm-builtin.ts

@@ -0,0 +1,246 @@
+import { existsSync, readdirSync } from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import * as ts from "typescript";
+import { LuaLibFeature, type Plugin, type TransformationContext } from "typescript-to-lua";
+import { scanEmittedRequires } from "./lua-require-scan";
+
+// `@defold-typescript/types/hsm` is the `index` module; every other module is a
+// subpath of it. Each lowers to a require of `defold_typescript_hsm.<module>`,
+// which the CLI writes under the output root the way it writes the timers
+// runtime.
+export const HSM_MODULE_SPECIFIER = "@defold-typescript/types/hsm";
+export const HSM_REQUIRE_ROOT = "defold_typescript_hsm";
+
+// The hsm modules whose Lua requires the lualib bundle. Every other module stays
+// plain Lua, so a program that imports none of these ships no bundle.
+export const HSM_LUALIB_MODULES: ReadonlySet<string> = new Set(["async"]);
+
+export interface HsmModule {
+  /** The source file's basename without `.ts`. */
+  readonly name: string;
+  /** What a user imports. */
+  readonly specifier: string;
+  /** What the import lowers to, and the module path the CLI writes it under. */
+  readonly requireName: string;
+}
+
+export function hsmRequireName(name: string): string {
+  return `${HSM_REQUIRE_ROOT}.${name}`;
+}
+
+function hsmSpecifier(name: string): string {
+  return name === "index" ? HSM_MODULE_SPECIFIER : `${HSM_MODULE_SPECIFIER}/${name}`;
+}
+
+// Same anchoring as the CLI's version lookup: the module URL, never
+// `import.meta.dir`, so the bundled `dist/index.js` under node still lands on
+// the package root.
+function defaultPackageRoot(): string {
+  return path.join(path.dirname(fileURLToPath(import.meta.url)), "..");
+}
+
+// The one filter the transpiler build (what ships in `dist/hsm/`) and the
+// compile (what reaches a project) both read, so a new module ships without
+// editing a list.
+export function hsmSourceFiles(dir: string): string[] {
+  return readdirSync(dir, { withFileTypes: true })
+    .filter(
+      (entry) =>
+        entry.isFile() &&
+        entry.name.endsWith(".ts") &&
+        !entry.name.endsWith(".test.ts") &&
+        !entry.name.endsWith(".d.ts"),
+    )
+    .map((entry) => entry.name)
+    .sort();
+}
+
+// The workspace sibling wins so a stale `dist/` never shadows the live source
+// inside this repo; a published install has only the `dist/hsm/` copy.
+export function resolveHsmSourceDir(packageRoot: string = defaultPackageRoot()): string | null {
+  const workspace = path.join(packageRoot, "..", "hsm", "src");
+  if (existsSync(workspace)) {
+    return workspace;
+  }
+  const shipped = path.join(packageRoot, "dist", "hsm");
+  if (existsSync(shipped)) {
+    return shipped;
+  }
+  return null;
+}
+
+export function requireHsmSourceDir(): string {
+  const dir = resolveHsmSourceDir();
+  if (dir === null) {
+    throw new Error(
+      "@defold-typescript/transpiler: the hsm source is missing from this install; reinstall @defold-typescript/transpiler.",
+    );
+  }
+  return dir;
+}
+
+let modules: readonly HsmModule[] | undefined;
+
+export function hsmModules(): readonly HsmModule[] {
+  modules ??= hsmSourceFiles(requireHsmSourceDir()).map((file) => {
+    const name = file.slice(0, -".ts".length);
+    return { name, specifier: hsmSpecifier(name), requireName: hsmRequireName(name) };
+  });
+  return modules;
+}
+
+/** The hsm module a require path names, or undefined for any other require. */
+function hsmModuleOfRequire(requirePath: string): HsmModule | undefined {
+  if (!requirePath.startsWith(`${HSM_REQUIRE_ROOT}.`)) {
+    return undefined;
+  }
+  return hsmModules().find((module) => module.requireName === requirePath);
+}
+
+export function isHsmRequireName(requirePath: string): boolean {
+  return hsmModuleOfRequire(requirePath) !== undefined;
+}
+
+function hsmModuleOfSpecifier(node: ts.Expression | undefined): HsmModule | undefined {
+  if (node === undefined || !ts.isStringLiteral(node)) {
+    return undefined;
+  }
+  const { text } = node;
+  if (text !== HSM_MODULE_SPECIFIER && !text.startsWith(`${HSM_MODULE_SPECIFIER}/`)) {
+    return undefined;
+  }
+  return hsmModules().find((module) => module.specifier === text);
+}
+
+function requiredHsmNames(lua: string): string[] {
+  const names: string[] = [];
+  for (const { path: requirePath } of scanEmittedRequires(lua)) {
+    const module = hsmModuleOfRequire(requirePath);
+    if (module !== undefined) {
+      names.push(module.name);
+    }
+  }
+  return names;
+}
+
+/**
+ * The hsm modules a program needs: every one its chunks require, plus every one
+ * those require in turn. Read from emitted Lua rather than the import AST, so an
+ * import TypeScript elides (`import type`, an unused binding) selects nothing.
+ * `compile` runs only when some chunk requires a module.
+ */
+export function hsmClosure(
+  chunks: Iterable<string>,
+  compile: () => Readonly<Record<string, string>>,
+): Record<string, string> | undefined {
+  const pending: string[] = [];
+  for (const lua of chunks) {
+    pending.push(...requiredHsmNames(lua));
+  }
+  if (pending.length === 0) {
+    return undefined;
+  }
+  const compiled = compile();
+  const selected: Record<string, string> = {};
+  for (let name = pending.pop(); name !== undefined; name = pending.pop()) {
+    const lua = compiled[name];
+    if (name in selected || lua === undefined) {
+      continue;
+    }
+    selected[name] = lua;
+    pending.push(...requiredHsmNames(lua));
+  }
+  return Object.fromEntries(Object.entries(selected).sort(([a], [b]) => a.localeCompare(b)));
+}
+
+// Rewrite an hsm specifier to its flat require, the way the timers lowering
+// does: `@NoResolution:` keeps TSTL from looking for a Lua source behind the
+// `.d.ts` the import type-checks against.
+function lowerSpecifier(module: HsmModule): ts.StringLiteral {
+  return ts.factory.createStringLiteral(`@NoResolution:${module.requireName}`);
+}
+
+type Lowered = ReturnType<TransformationContext["superTransformStatements"]>;
+
+function loweredAnything(lowered: Lowered): boolean {
+  return Array.isArray(lowered) ? lowered.length > 0 : lowered !== undefined;
+}
+
+// The same resolver verdict TSTL's own import transform uses to decide which
+// bindings it requires, so the bundle and the emitted require cannot disagree.
+// Takes the original node: the resolver has no binding for a synthesized copy.
+function importSurvivesElision(
+  node: ts.ImportDeclaration,
+  context: TransformationContext,
+): boolean {
+  const clause = node.importClause;
+  if (clause === undefined) {
+    return true;
+  }
+  const { resolver } = context;
+  if (clause.name !== undefined && resolver.isReferencedAliasDeclaration(clause)) {
+    return true;
+  }
+  const bindings = clause.namedBindings;
+  if (bindings === undefined) {
+    return false;
+  }
+  if (ts.isNamespaceImport(bindings)) {
+    return resolver.isReferencedAliasDeclaration(bindings);
+  }
+  return bindings.elements.some((element) => resolver.isReferencedAliasDeclaration(element));
+}
+
+// TSTL writes `lualib_bundle.lua` only for features a user chunk registers; it
+// never sees the hsm module's own source, which the CLI writes beside the build.
+// An elided import (type-only, unused) registers nothing. The import visitor
+// passes the resolver's elision verdict, because TSTL's import transform always
+// returns nothing and queues its require on the scope instead; the export
+// visitor passes whether its lowered statements are non-empty, because TSTL's
+// export transform returns them.
+function registerLualib(module: HsmModule, emitted: boolean, context: TransformationContext): void {
+  if (emitted && HSM_LUALIB_MODULES.has(module.name)) {
+    context.usedLuaLibFeatures.add(LuaLibFeature.Promise);
+    context.usedLuaLibFeatures.add(LuaLibFeature.New);
+  }
+}
+
+export const hsmLoweringPlugin: Plugin = {
+  visitors: {
+    [ts.SyntaxKind.ImportDeclaration]: (node, context) => {
+      const module = hsmModuleOfSpecifier(node.moduleSpecifier);
+      if (module === undefined) {
+        return context.superTransformStatements(node);
+      }
+      registerLualib(module, importSurvivesElision(node, context), context);
+      return context.superTransformStatements(
+        ts.factory.updateImportDeclaration(
+          node,
+          node.modifiers,
+          node.importClause,
+          lowerSpecifier(module),
+          node.attributes,
+        ),
+      );
+    },
+    [ts.SyntaxKind.ExportDeclaration]: (node, context) => {
+      const module = hsmModuleOfSpecifier(node.moduleSpecifier);
+      if (module === undefined) {
+        return context.superTransformStatements(node);
+      }
+      const lowered = context.superTransformStatements(
+        ts.factory.updateExportDeclaration(
+          node,
+          node.modifiers,
+          node.isTypeOnly,
+          node.exportClause,
+          lowerSpecifier(module),
+          node.attributes,
+        ),
+      );
+      registerLualib(module, loweredAnything(lowered), context);
+      return lowered;
+    },
+  },
+};

package/src/index.ts

@@ -20,6 +20,16 @@ export {
   guiScriptResourceOf,
   guiScriptResourcesOf,
 } from "./gui-node-index";
+export type { HsmModule } from "./hsm-builtin";
+export {
+  HSM_MODULE_SPECIFIER,
+  HSM_REQUIRE_ROOT,
+  hsmModules,
+  hsmRequireName,
+  hsmSourceFiles,
+  requireHsmSourceDir,
+  resolveHsmSourceDir,
+} from "./hsm-builtin";
 export { buildInputActionIndex } from "./input-action-index";
 export type {
   ExtensionDependency,
@@ -99,7 +109,7 @@ export type {
   TranspileProjectResult,
   TranspileResult,
 } from "./transpile";
-export { transpile, transpileProject } from "./transpile";
+export { compileHsmModules, transpile, transpileProject } from "./transpile";
 export type { ClassifiedSlot } from "./url-address-slots";
 export {
   addressClassOfArgument,

package/src/session.ts

@@ -5,6 +5,7 @@ import * as ts from "typescript";
 import * as tstl from "typescript-to-lua";
 import { type CompanionEmitter, createCompanionEmitPlugin } from "./companion-emit";
 import { editorScriptErasurePlugin } from "./editor-script-erasure";
+import { hsmLoweringPlugin } from "./hsm-builtin";
 import { lifecycleErasurePlugin } from "./lifecycle-erasure";
 import { messageDispatchLoweringPlugin } from "./message-dispatch-lowering";
 import { messageGuardLoweringPlugin } from "./message-guard-lowering";
@@ -44,6 +45,7 @@ function compilerOptions(companionEmit: CompanionEmitter): tstl.CompilerOptions
       { plugin: windowEventGuardLoweringPlugin },
       { plugin: messageDispatchLoweringPlugin },
       { plugin: timersLoweringPlugin },
+      { plugin: hsmLoweringPlugin },
       { plugin: typeApplicationErasurePlugin },
       { plugin: companionEmit.plugin },
     ],

package/src/transpile.ts

@@ -9,7 +9,16 @@ import {
   findDirectGoPropertyCalls,
   GO_PROPERTY_DIRECT_CALL_MESSAGE,
 } from "./go-property-direct-call";
+import {
+  HSM_LUALIB_MODULES,
+  HSM_REQUIRE_ROOT,
+  hsmClosure,
+  hsmLoweringPlugin,
+  hsmModules,
+  requireHsmSourceDir,
+} from "./hsm-builtin";
 import { lifecycleErasurePlugin } from "./lifecycle-erasure";
+import { scanEmittedRequires } from "./lua-require-scan";
 import { messageDispatchLoweringPlugin } from "./message-dispatch-lowering";
 import { messageGuardLoweringPlugin } from "./message-guard-lowering";
 import { importsTimersModule, timersLoweringPlugin } from "./timers-lowering";
@@ -63,6 +72,13 @@ export interface TranspileProjectResult {
   // must write this to the output root. Present only when a user file imports
   // the module (pay-for-use).
   readonly timersRuntime?: string;
+  // The compiled `hsm` modules the program requires, keyed by module name
+  // (`index`, `defold`, ...). An hsm import lowers to
+  // `require("defold_typescript_hsm.<module>")`, so the CLI must write each one
+  // under the output root. Present only when some chunk requires one
+  // (pay-for-use); never carries a source map, since the `.ts` it would point
+  // at is not in the user's project.
+  readonly hsmModules?: Readonly<Record<string, string>>;
 }
 
 function flattenDiagnosticMessage(
@@ -180,6 +196,14 @@ function buildAmbientFiles(): Record<string, string> {
       "",
     ].join("\n"),
   };
+  // The generated `hsm` declarations, at the paths the package `exports` map
+  // names, so the build types user code against the same `.d.ts` tsserver
+  // reads. hsm's own `.ts` compiles separately, in `compileHsmModules`.
+  for (const entry of readdirSync(path.join(TYPES_PKG_ROOT, "hsm"))) {
+    if (entry.endsWith(".d.ts")) {
+      files[`node_modules/@defold-typescript/types/hsm/${entry}`] = readAmbient(`hsm/${entry}`);
+    }
+  }
   // Seed the Lua 5.1 standard library (math/os/string/table/coroutine + base
   // globals) so user code can call e.g. `math.randomseed(os.time())`.
   Object.assign(files, collectLuaTypesClosure());
@@ -287,6 +311,8 @@ export function collectOutputs(
     }
   }
 
+  const hsm = hsmClosure([...Object.values(lua), ...Object.values(companions)], compileHsmModules);
+
   return {
     lua,
     sourceMaps,
@@ -294,38 +320,48 @@ export function collectOutputs(
     ...(Object.keys(companions).length > 0 ? { companions } : {}),
     ...(lualib !== undefined ? { lualib } : {}),
     ...(timersImported ? { timersRuntime: TIMERS_RUNTIME } : {}),
+    ...(hsm !== undefined ? { hsmModules: hsm } : {}),
   };
 }
 
+function transpileVirtual(
+  files: Readonly<Record<string, string>>,
+  companionEmit: ReturnType<typeof createCompanionEmitPlugin>,
+): tstl.TranspileVirtualProjectResult {
+  return tstl.transpileVirtualProject(
+    { ...AMBIENT_FILES, ...files },
+    {
+      luaTarget: tstl.LuaTarget.Lua51,
+      sourceMap: true,
+      // Already-resolved CompilerOptions, so the value is the lib file name, not
+      // the tsconfig spelling "ES2022". Drops the default `lib.dom`, whose
+      // `declare var window` shadows Defold's `window` namespace.
+      lib: ["lib.es2022.d.ts"],
+      // Don't cross-check the seeded ambient .d.ts surface against itself; we only
+      // care about diagnostics on user files (mirrors the editor's skipLibCheck).
+      skipLibCheck: true,
+      // Defold scripts are not OO: free helper functions never receive a context,
+      // so suppress TSTL's implicit `self` parameter and the `_G` call-site filler.
+      noImplicitSelf: true,
+      luaPlugins: [
+        { plugin: lifecycleErasurePlugin },
+        { plugin: editorScriptErasurePlugin },
+        { plugin: messageGuardLoweringPlugin },
+        { plugin: windowEventGuardLoweringPlugin },
+        { plugin: messageDispatchLoweringPlugin },
+        { plugin: timersLoweringPlugin },
+        { plugin: hsmLoweringPlugin },
+        { plugin: typeApplicationErasurePlugin },
+        { plugin: companionEmit.plugin },
+      ],
+    },
+  );
+}
+
 export function transpileProject(input: TranspileProjectInput): TranspileProjectResult {
   const userKeys = new Set(Object.keys(input.files));
-  const merged: Record<string, string> = { ...AMBIENT_FILES, ...input.files };
   const companionEmit = createCompanionEmitPlugin();
-
-  const result = tstl.transpileVirtualProject(merged, {
-    luaTarget: tstl.LuaTarget.Lua51,
-    sourceMap: true,
-    // Already-resolved CompilerOptions, so the value is the lib file name, not
-    // the tsconfig spelling "ES2022". Drops the default `lib.dom`, whose
-    // `declare var window` shadows Defold's `window` namespace.
-    lib: ["lib.es2022.d.ts"],
-    // Don't cross-check the seeded ambient .d.ts surface against itself; we only
-    // care about diagnostics on user files (mirrors the editor's skipLibCheck).
-    skipLibCheck: true,
-    // Defold scripts are not OO: free helper functions never receive a context,
-    // so suppress TSTL's implicit `self` parameter and the `_G` call-site filler.
-    noImplicitSelf: true,
-    luaPlugins: [
-      { plugin: lifecycleErasurePlugin },
-      { plugin: editorScriptErasurePlugin },
-      { plugin: messageGuardLoweringPlugin },
-      { plugin: windowEventGuardLoweringPlugin },
-      { plugin: messageDispatchLoweringPlugin },
-      { plugin: timersLoweringPlugin },
-      { plugin: typeApplicationErasurePlugin },
-      { plugin: companionEmit.plugin },
-    ],
-  });
+  const result = transpileVirtual(input.files, companionEmit);
 
   return collectOutputs(
     result.transpiledFiles,
@@ -335,6 +371,62 @@ export function transpileProject(input: TranspileProjectInput): TranspileProject
   );
 }
 
+let compiledHsm: Readonly<Record<string, string>> | undefined;
+
+function requiresLualib(lua: string): boolean {
+  return scanEmittedRequires(lua).some(({ path: requirePath }) => requirePath === "lualib_bundle");
+}
+
+/**
+ * Every `hsm` module compiled to Lua, keyed by module name, once per process.
+ * The source is the shipped TypeScript, never a prebuilt Lua copy, so the Lua a
+ * project gets always comes from the compiler that builds it. Each file sits at
+ * `defold_typescript_hsm/<module>.ts`, so a require between modules is already
+ * spelled the way the user's lowered imports are. Throws when the compile
+ * reports anything, or when a module outside `HSM_LUALIB_MODULES` needs the
+ * lualib bundle.
+ */
+export function compileHsmModules(): Readonly<Record<string, string>> {
+  if (compiledHsm !== undefined) {
+    return compiledHsm;
+  }
+  const sourceDir = requireHsmSourceDir();
+  const nameByKey = new Map<string, string>();
+  const files: Record<string, string> = {};
+  for (const { name } of hsmModules()) {
+    const key = `${HSM_REQUIRE_ROOT}/${name}.ts`;
+    nameByKey.set(key, name);
+    files[key] = readFileSync(path.join(sourceDir, `${name}.ts`), "utf8");
+  }
+  const result = transpileVirtual(files, createCompanionEmitPlugin());
+
+  const problems = result.diagnostics.map(
+    (d) => `${d.file?.fileName ?? "(project)"}: ${flattenDiagnosticMessage(d.messageText)}`,
+  );
+  const lua: Record<string, string> = {};
+  for (const file of result.transpiledFiles) {
+    if (isLualibBundle(file)) {
+      continue;
+    }
+    const source = file.sourceFiles.find((s) => nameByKey.has(s.fileName));
+    const name = source === undefined ? undefined : nameByKey.get(source.fileName);
+    if (name === undefined || typeof file.lua !== "string") {
+      continue;
+    }
+    if (!HSM_LUALIB_MODULES.has(name) && requiresLualib(file.lua)) {
+      problems.push(`the compiled ${name} module requires lualib_bundle`);
+    }
+    lua[name] = file.lua;
+  }
+  if (problems.length > 0) {
+    throw new Error(
+      `@defold-typescript/transpiler: the hsm library does not compile:\n  ${problems.join("\n  ")}`,
+    );
+  }
+  compiledHsm = lua;
+  return lua;
+}
+
 export function transpile(source: string): TranspileResult {
   const project = transpileProject({ files: { "main.ts": source } });
   return {