@zmdb/compiler 1.0.0-beta.1

package/src/reflect/session.ts

@@ -0,0 +1,276 @@
+// The one place that talks to the TypeScript compiler.
+//
+// `typescript@7` is the Go compiler with a thin JS client: `import('typescript')`
+// resolves to exactly two keys (`version`, `versionMajorMinor`) and the real surface
+// lives behind subpath exports. The checker is `typescript/unstable/sync`, and every
+// call round-trips to a `tsgo` child process over a synchronous pipe.
+//
+// Two consequences shape this file:
+//
+//  1. **One session per build, not one per type.** Spawning the server and loading a
+//     project is the expensive part; a checker call is cheap. Every consumer takes a
+//     `ReflectSession` and holds it for the whole run.
+//  2. **`close()` is not optional.** The child process outlives the import if nobody
+//     closes it, which in a test runner means a hung worker. `using` works, and
+//     `withSession` is there for the cases that cannot use it.
+//
+// `unstable` in the specifier is the compiler's word, not ours: the API is not
+// covered by TypeScript's stability guarantee, so it is contained here rather than
+// spread over the reflection and the emitter.
+
+import { API } from 'typescript/unstable/sync';
+import type { Checker, Diagnostic, Program } from 'typescript/unstable/sync';
+
+/** A parsed source file, as the checker's client models one. */
+export type SourceFileHandle = NonNullable<ReturnType<Program['getSourceFile']>>;
+
+export interface SessionOptions {
+  /** Absolute path to the `tsconfig.json` that defines the program. */
+  readonly project: string;
+  /** Working directory for module resolution. Defaults to the project's directory. */
+  readonly cwd?: string;
+}
+
+interface ParsedProjectConfig {
+  readonly fileNames: readonly string[];
+}
+
+type PrepareProject = (config: ParsedProjectConfig) => string;
+const PREPARED_OPEN = Symbol('preparedOpen');
+
+/** What a snapshot update was for. The session keeps the log; see `updates`. */
+export type SessionUpdate = 'open' | 'refresh' | 'invalidate';
+
+// The three things a watcher can report about a file. Mutable arrays, and not because
+// anything here mutates them: the compiler's own `FileChangeSummary` declares them mutable,
+// and under `exactOptionalPropertyTypes` a `readonly string[]` will not go in.
+interface FileEdits {
+  changed?: string[];
+  created?: string[];
+  deleted?: string[];
+}
+
+/**
+ * How many compiler servers this process has started.
+ *
+ * REQ-TF-11 asks for one `API` instance per build, which is the difference between a
+ * build that loads the project once and one that loads it per file. A claim like that
+ * needs a number behind it, so the counter lives here and the build-budget test reads
+ * it.
+ */
+let apiInstances = 0;
+
+export function apiInstanceCount(): number {
+  return apiInstances;
+}
+
+/**
+ * An open compiler session: one server process, one loaded project, one checker.
+ *
+ * ```ts
+ * using session = ReflectSession.open({ project: '/abs/tsconfig.json' });
+ * const ir = irFromType({ checker: session.checker, location: sf }, type);
+ * ```
+ */
+export class ReflectSession implements Disposable {
+  readonly project: string;
+
+  #api: API;
+  #program: Program;
+  #checker: Checker;
+  #closed = false;
+  readonly #updates: SessionUpdate[] = ['open'];
+
+  private constructor(api: API, project: string, program: Program, checker: Checker) {
+    this.#api = api;
+    this.project = project;
+    this.#program = program;
+    this.#checker = checker;
+  }
+
+  /**
+   * The checker and program come from the *current* snapshot, so they are getters
+   * rather than fields: after `refresh` the old pair belongs to a disposed snapshot,
+   * and a caller holding one would be reading a program that no longer exists.
+   */
+  get checker(): Checker {
+    return this.#checker;
+  }
+
+  get program(): Program {
+    return this.#program;
+  }
+
+  /** Every snapshot update this session has performed, in order. See `SessionUpdate`. */
+  get updates(): readonly SessionUpdate[] {
+    return this.#updates;
+  }
+
+  static open(options: SessionOptions): ReflectSession {
+    return ReflectSession[PREPARED_OPEN](options);
+  }
+
+  static [PREPARED_OPEN](options: SessionOptions, prepare?: PrepareProject): ReflectSession {
+    const cwd = options.cwd ?? options.project.replace(/[/\\][^/\\]*$/, '');
+    apiInstances++;
+    const api = new API({ cwd });
+    try {
+      const projectPath =
+        prepare === undefined
+          ? options.project
+          : prepare({ fileNames: api.parseConfigFile(options.project).fileNames });
+      // `getProjects()` is empty rather than throwing when the config does not parse,
+      // and an empty project list would otherwise surface much later as "every type is
+      // `unsupported`" — a wrong answer dressed up as a supported one.
+      const project = api.updateSnapshot({ openProjects: [projectPath] }).getProjects()[0];
+      if (!project) throw new Error(`could not load a TypeScript project from ${projectPath}`);
+      return new ReflectSession(api, projectPath, project.program, project.checker);
+    } catch (error: unknown) {
+      api.close();
+      throw error;
+    }
+  }
+
+  /**
+   * Pick up edits to `changed` without reloading the project.
+   *
+   * This is the whole of watch-mode support, and the reason it is one method rather
+   * than "open a new session" is cost: `openProjects` re-reads the config and re-walks
+   * the import graph, which is the expensive half of a build. `fileChanges` re-checks
+   * the files that changed. A watch that reopened the project per keystroke would make
+   * the AOT path slower than the runtime one it replaces.
+   */
+  refresh(changed: readonly string[]): void {
+    this.#files({ changed: [...changed] });
+  }
+
+  /**
+   * Tell the program about files that did not exist when it loaded.
+   *
+   * A `changed` notification for a file the program has never seen is a no-op — measured:
+   * `getSourceFile` keeps returning `undefined` — so a new module has to arrive as
+   * `created` or it stays invisible for the rest of the build. That is exactly the shape
+   * of "add a file in watch mode", so it is not an edge case.
+   */
+  created(files: readonly string[]): void {
+    this.#files({ created: [...files] });
+  }
+
+  deleted(files: readonly string[]): void {
+    this.#files({ deleted: [...files] });
+  }
+
+  #files(changes: FileEdits): void {
+    if (Object.values(changes).every(names => names.length === 0)) return;
+    this.#update('refresh', { fileChanges: changes });
+  }
+
+  /** For the cases a watcher cannot describe — a config change, a new dependency. */
+  invalidateAll(): void {
+    this.#update('invalidate', { fileChanges: { invalidateAll: true } });
+  }
+
+  #update(kind: SessionUpdate, params: Parameters<API['updateSnapshot']>[0]): void {
+    if (this.#closed) throw new Error('the compiler session is closed');
+    // Deliberately no `openProjects` here: the open is ref-counted and persists across
+    // snapshots, so naming it again would both leak a reference and reload the project.
+    const snapshot = this.#api.updateSnapshot(params);
+    const project = snapshot.getProject(this.project) ?? snapshot.getProjects()[0];
+    if (!project) throw new Error(`the TypeScript project ${this.project} disappeared from the snapshot`);
+    this.#program = project.program;
+    this.#checker = project.checker;
+    this.#updates.push(kind);
+  }
+
+  /** The parsed file, or `undefined` when it is not part of the program. */
+  sourceFile(fileName: string): SourceFileHandle | undefined {
+    return this.#program.getSourceFile(fileName);
+  }
+
+  /**
+   * Every file in the program, dependencies and lib files included.
+   *
+   * Project compilation needs this and the plugin does not: a bundler hands over one module
+   * at a time, so the plugin never has to ask what the project contains. A project compile
+   * covers the whole build, so "which files might have a call site in them" is its first question.
+   *
+   * `getSourceFiles()` — the name the old compiler used — is not on the client at all;
+   * only the names marshal, and the caller has to filter `node_modules` and `.d.ts` for
+   * itself.
+   */
+  sourceFileNames(): readonly string[] {
+    return this.#program.getSourceFileNames();
+  }
+
+  /**
+   * The project's resolved compiler options.
+   *
+   * Read for one thing: whether a generated module may import `./models.ts` with the
+   * extension it has on disk. Guessing wrong writes a project that does not compile,
+   * which is a worse failure than refusing.
+   */
+  compilerOptions(): Readonly<Record<string, unknown>> {
+    // boundary: the client types this as `CompilerOptions`, an interface with a declared
+    // property per known flag. Widening it to a record is the safe direction — every value
+    // becomes `unknown` and the caller has to test what it reads — and it is the only way to
+    // ask about a flag by name, which is what a caller that only wants one flag needs.
+    return this.#program.getCompilerOptions() as Readonly<Record<string, unknown>>;
+  }
+
+  /**
+   * Semantic diagnostics for one file. The reflection reads types, and a type in a
+   * file that does not compile is a guess, so callers check this before trusting
+   * anything derived from it.
+   */
+  diagnostics(fileName: string): readonly Diagnostic[] {
+    return this.#program.getSemanticDiagnostics(fileName);
+  }
+
+  close(): void {
+    if (this.#closed) return;
+    this.#closed = true;
+    this.#api.close();
+  }
+
+  [Symbol.dispose](): void {
+    this.close();
+  }
+}
+
+/**
+ * Open one session after preparing a disposable project from the compiler's parsed
+ * root-file inventory.
+ *
+ * Project compilation uses this to admit generated witness files even when the consumer
+ * config has an explicit `files`/`include` list. The preparation and project open share one
+ * API instance, so this remains one compiler session rather than a discovery pass followed
+ * by a second compiler process.
+ *
+ * This helper is intentionally not re-exported from `@zmdb/compiler/reflect`.
+ */
+export function openPreparedSession(options: SessionOptions, prepare: PrepareProject): ReflectSession {
+  return ReflectSession[PREPARED_OPEN](options, prepare);
+}
+
+/**
+ * The concrete source-file set selected by a tsconfig.
+ *
+ * Config loading needs this narrower fact without retaining a compiler process:
+ * schema globs are allowed to select files only when the configured project
+ * includes them. Copy the names before closing because the program belongs to
+ * the session snapshot.
+ */
+export function projectSourceFileNames(project: string): readonly string[] {
+  using session = ReflectSession.open({ project });
+  return [...session.sourceFileNames()];
+}
+
+/** `try`/`finally` around a session, for callers that cannot use `using`. */
+export function withSession<T>(options: SessionOptions, fn: (session: ReflectSession) => T): T {
+  const session = ReflectSession.open(options);
+  try {
+    return fn(session);
+  } finally {
+    session.close();
+  }
+}

package/src/testing/index.ts

@@ -0,0 +1,298 @@
+// `@zmdb/compiler/testing` — tagged schema values, without a transform step.
+//
+// `schemaOf<User>()` is compiled away by the transform, and it has no runtime: a type
+// argument does not survive to runtime, so a build that skipped the transform gets a thrown
+// error rather than a plausible-looking empty schema. That is the right behaviour in an
+// application and an awkward one in a test, because a unit test usually runs off the
+// TypeScript source with no bundler in the way — `vitest`, `node --strip-types`, `tsx`. Ask
+// for the schema there and you get the error, correctly, and no way forward.
+//
+// So this does through the compiler API what the transform does during a build: open the
+// project, read exported tagged interfaces, and turn their IR into schema values. Tests can
+// name individual exports with `schemasFrom`; build tools whose config already resolved a
+// concrete file set use `schemasFromFiles`. Both return what the transform would have inlined
+// — literally the same `schemaFromIR(schemaIrFromType(...))` expression — rather than a
+// stand-in for the shipped path.
+//
+// ```ts
+// import { schemasFrom } from '@zmdb/compiler/testing';
+//
+// export interface User extends Table<'users'> {
+//   id: number & Sql<'integer'> & Serial & PrimaryKey;
+//   email: string & Sql<'varchar'> & Length<255> & Unique;
+// }
+//
+// const { User: users } = schemasFrom(import.meta.url, ['User']);
+// ```
+//
+// The interfaces can live in the test file itself, which is the point: a fixture two
+// directories away is a fixture nobody reads. They do have to be `export`ed, because the
+// module's export table is how a name is resolved to a symbol.
+//
+// ## What it costs
+//
+// One compiler session per call — about 80ms to load a package-sized project, and about 3ms
+// to resolve a type out of it. So a test should call once at module scope with every name it
+// needs, while a command should pass its whole configured file set once. The session is
+// closed before the call returns, because an open one holds a child process and a process
+// that leaks those hangs on exit.
+//
+// It does check the module's own diagnostics first, which costs about 6ms and earns it back
+// the first time somebody mistypes an import. A type read out of a file that does not compile
+// comes back as an error type, and the reflection reports an error type as "the checker could
+// not resolve this type" — true, unhelpful, and repeated once per column. The compile error is
+// the thing that happened, so that is what gets raised.
+
+import { existsSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { type CoreSchema, type TaggedSchema } from '@zmdb/schema';
+import { schemaFromIR, type SchemaIR } from '@zmdb/schema/ir';
+import { type NamingStrategy } from '@zmdb/schema/naming';
+
+import { schemaIrFromType, type ReflectDiagnostic, type ReflectOptions } from '../reflect/index.js';
+import { ReflectSession } from '../reflect/session.js';
+
+export interface SchemasFromOptions {
+  /**
+   * The `tsconfig.json` to read the module from. Defaults to the nearest one at or above the
+   * module's own directory, which is the right answer for a test inside its package.
+   */
+  readonly project?: string | undefined;
+  /**
+   * What to do with a reflection that had to refuse something. The default throws, because a
+   * schema with a column quietly missing is the failure mode this whole design exists to
+   * prevent; pass a function to inspect them instead.
+   */
+  readonly onDiagnostics?: ((diagnostics: readonly ReflectDiagnostic[]) => void) | undefined;
+  /** The already-resolved build-time naming strategy. */
+  readonly naming?: NamingStrategy | undefined;
+}
+
+export interface SchemasFromFilesOptions extends SchemasFromOptions {
+  /** The project that owns every selected declaration file. */
+  readonly project: string;
+}
+
+function reflectOptions(options: SchemasFromOptions): ReflectOptions | undefined {
+  return options.naming === undefined ? undefined : { naming: options.naming };
+}
+
+/**
+ * The schema values for exported tagged interfaces in one module.
+ *
+ * `module` is a path or a `file:` URL — `import.meta.url` for the calling test file, which is
+ * the usual case. The result is keyed by interface name, so it destructures.
+ */
+export function schemasFrom<const Names extends readonly string[]>(
+  module: string,
+  names: Names,
+  options?: SchemasFromOptions,
+): { [Name in Names[number]]: CoreSchema<string> };
+/**
+ * The same, with each schema remembering the type it came from.
+ *
+ * The reflection knows which interface it read, but a function cannot return a different type
+ * per string it was handed, so the mapping is stated: `schemasFrom<{ User: User }>(…)`. What
+ * comes back is a `TaggedSchema<User>`, exactly what `schemaOf<User>()` would have been, which
+ * is what makes `Entity<…>`, `CreateDTO<…>` and a typed repository work off it.
+ *
+ * The name appears twice, once as a type and once as a string. That is the honest amount: the
+ * string is what gets looked up at runtime and the type is what the compiler needs, and the
+ * constraint ties them together so a typo in either is an error rather than an `undefined`.
+ * Tests that only read the schema as data — IR, OpenAPI, seeding — want the erased overload
+ * above and should not pay the extra line.
+ */
+export function schemasFrom<Types extends Record<string, object>>(
+  module: string,
+  names: readonly (keyof Types & string)[],
+  options?: SchemasFromOptions,
+): { [Name in keyof Types]: TaggedSchema<Types[Name]> };
+export function schemasFrom(
+  module: string,
+  names: readonly string[],
+  options: SchemasFromOptions = {},
+): Record<string, CoreSchema<string>> {
+  const irs: Record<string, SchemaIR> = schemaIrsFrom(module, names, options);
+  const schemas: Record<string, CoreSchema<string>> = {};
+  for (const [name, ir] of Object.entries(irs)) schemas[name] = schemaFromIR(ir);
+  return schemas;
+}
+
+/**
+ * Every exported tagged table declaration in a concrete file set.
+ *
+ * Config loading has already expanded and project-checked these files. This
+ * bridge keeps the remaining compiler work in the reflection library: one
+ * session for the project, one pass over each module's exports, and the same
+ * `schemaFromIR` conversion used by `schemaOf<T>()` and `schemasFrom()`.
+ */
+export function schemasFromFiles(
+  files: readonly string[],
+  options: SchemasFromFilesOptions,
+): readonly CoreSchema<string>[] {
+  if (files.length === 0) throw new Error('the configured schema file set is empty');
+
+  using session = ReflectSession.open({ project: options.project });
+  const schemas = new Map<string, CoreSchema<string>>();
+  const diagnostics: ReflectDiagnostic[] = [];
+
+  for (const file of files.toSorted()) {
+    const sourceFile = session.sourceFile(file);
+    if (!sourceFile) {
+      throw new Error(
+        `${file} is not part of ${options.project}; the configured schema files must belong to the project`,
+      );
+    }
+
+    const broken = session.diagnostics(file);
+    if (broken.length > 0) {
+      throw new Error(
+        `${file} does not compile, so its table declarations cannot be read (${broken.length} diagnostic(s)):\n` +
+          broken
+            .slice(0, 5)
+            .map(one => `  TS${String(one.code)}: ${one.text}`)
+            .join('\n'),
+      );
+    }
+
+    const moduleSymbol = session.checker.getSymbolAtLocation(sourceFile);
+    if (!moduleSymbol) throw new Error(`${file} has no module symbol, so it exports nothing to read`);
+    const exported = session.checker
+      .getExportsOfModule(moduleSymbol)
+      .toSorted((left, right) => left.name.localeCompare(right.name));
+
+    for (const symbol of exported) {
+      const type = session.checker.getDeclaredTypeOfSymbol(symbol);
+      const reflected = schemaIrFromType(session.checker, type, sourceFile, reflectOptions(options));
+      const isTable = reflected.diagnostics.every(diagnostic => !diagnostic.reason.includes("no Table<'name'> tag"));
+      if (!isTable) continue;
+
+      diagnostics.push(...reflected.diagnostics);
+      const schema = schemaFromIR(reflected.ir);
+      const previous = schemas.get(schema.ir.table);
+      if (previous !== undefined) {
+        if (JSON.stringify(previous.ir) !== JSON.stringify(schema.ir)) {
+          throw new Error(`configured schema files export conflicting declarations for table ${schema.ir.table}`);
+        }
+        continue;
+      }
+      schemas.set(schema.ir.table, schema);
+    }
+  }
+
+  if (diagnostics.length > 0) {
+    if (options.onDiagnostics) options.onDiagnostics(diagnostics);
+    else {
+      throw new Error(
+        `the reflection refused ${diagnostics.length} thing(s) in the configured schema files:\n` +
+          diagnostics.map(one => `  ${one.path ? `${one.path}: ` : ''}${one.reason}`).join('\n'),
+      );
+    }
+  }
+  if (schemas.size === 0) {
+    throw new Error(`the configured schema files export no tagged table declarations: ${files.toSorted().join(', ')}`);
+  }
+  return [...schemas.values()].toSorted((left, right) => left.ir.table.localeCompare(right.ir.table));
+}
+
+/**
+ * The same, stopping at the IR.
+ *
+ * Separate because the IR is the spine and a test about the front-end wants to see it
+ * directly — the schema value is one conversion further on, and `schemaFromIR` is not the
+ * thing under test.
+ */
+export function schemaIrsFrom<const Names extends readonly string[]>(
+  module: string,
+  names: Names,
+  options: SchemasFromOptions = {},
+): { [Name in Names[number]]: SchemaIR } {
+  const file = module.startsWith('file:') ? fileURLToPath(module) : resolve(module);
+  const project = options.project ?? nearestProject(file);
+
+  using session = ReflectSession.open({ project });
+  const sourceFile = session.sourceFile(file);
+  if (!sourceFile) {
+    throw new Error(
+      `${file} is not part of ${project}. The tagged interfaces have to be in the program for ` +
+        'the checker to have a declared type for them.',
+    );
+  }
+
+  // Before anything is read off the type: a file that does not compile has error types in it,
+  // and an error type reflects as a refusal per column rather than as the one problem it is.
+  const broken = session.diagnostics(file);
+  if (broken.length > 0) {
+    throw new Error(
+      `${file} does not compile, so its types cannot be read (${broken.length} diagnostic(s)):\n` +
+        broken
+          .slice(0, 5)
+          .map(one => `  TS${String(one.code)}: ${one.text}`)
+          .join('\n'),
+    );
+  }
+
+  const { checker } = session;
+  const moduleSymbol = checker.getSymbolAtLocation(sourceFile);
+  if (!moduleSymbol) throw new Error(`${file} has no module symbol, so it exports nothing to read`);
+  const exported = new Map(checker.getExportsOfModule(moduleSymbol).map(symbol => [symbol.name, symbol]));
+
+  const irs: Record<string, SchemaIR> = {};
+  const diagnostics: ReflectDiagnostic[] = [];
+  for (const name of names) {
+    const symbol = exported.get(name);
+    if (!symbol) {
+      // Naming the exports is worth the line: the usual cause is a missing `export`, and the
+      // message "User is not exported" is not obviously that when the interface is right there.
+      throw new Error(
+        `${file} exports no \`${name}\`. It has to be \`export interface ${name}\` — the module's ` +
+          `export table is how the name is resolved. Exports found: ${[...exported.keys()].join(', ') || 'none'}.`,
+      );
+    }
+    const type = checker.getDeclaredTypeOfSymbol(symbol);
+    // One reflector per name, which `schemaIrFromType` gives us: the node budget and the
+    // helper-name table are per-reflection state, and sharing them across unrelated tables
+    // would make one table's refusals show up against another's.
+    const result = schemaIrFromType(checker, type, sourceFile, reflectOptions(options));
+    irs[name] = result.ir;
+    diagnostics.push(...result.diagnostics);
+  }
+
+  if (diagnostics.length > 0) {
+    if (options.onDiagnostics) options.onDiagnostics(diagnostics);
+    else {
+      throw new Error(
+        `the reflection refused ${diagnostics.length} thing(s) in ${file}:\n` +
+          diagnostics.map(one => `  ${one.path ? `${one.path}: ` : ''}${one.reason}`).join('\n'),
+      );
+    }
+  }
+
+  // boundary: `irs` is filled in by the loop above, one key per member of `names`, and the
+  // loop throws rather than skipping when a name is not exported — so every key the return
+  // type promises is present by the time this runs. What the assertion buys is the *literal*
+  // keys: built from a `Record<string, SchemaIR>` because the names arrive as values, and no
+  // amount of building it differently makes the compiler read them back off the array.
+  return irs as { [Name in Names[number]]: SchemaIR };
+}
+
+/**
+ * The nearest `tsconfig.json` at or above a file.
+ *
+ * The same rule `tsc` uses for a bare invocation, and the same one the codegen CLI documents,
+ * so a caller who has not thought about it gets the project they would have guessed.
+ */
+function nearestProject(file: string): string {
+  let directory = dirname(file);
+  for (;;) {
+    const candidate = join(directory, 'tsconfig.json');
+    if (existsSync(candidate)) return candidate;
+    const parent = dirname(directory);
+    if (parent === directory) {
+      throw new Error(`no tsconfig.json at or above ${file}; pass \`project\` explicitly`);
+    }
+    directory = parent;
+  }
+}