@distilled.cloud/core 0.30.3 → 1.0.0-rc.1

package/src/codegen/cli.ts

@@ -0,0 +1,268 @@
+/**
+ * The generic generator CLI harness (dev-time only).
+ *
+ * Owns the smithy-model pipeline every SDK package shares: scan the model
+ * directory (plus optional hand-authored manual specs), apply the RFC-6902
+ * patch chain (`patches/<resource>/*.json`, `*.manual.json` last; stale
+ * targets warn, malformed patches fail the run), compile each model through
+ * {@link generateService} with the provider's {@link SdkSpec}, write the
+ * service modules and the namespaced barrel.
+ *
+ * A provider's `scripts/generate.ts` is: trait consts + an SdkSpec + a
+ * `runGeneratorCli` call.
+ */
+import { BunRuntime, BunServices } from "@effect/platform-bun";
+import { Console, Effect } from "effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import { Flag } from "effect/unstable/cli";
+import { Command } from "effect/unstable/cli";
+import {
+  applyOperation,
+  isStaleTargetError,
+  type PatchFile,
+} from "../json-patch.ts";
+import { barrel } from "./emit.ts";
+import { formatGenerated } from "./format.ts";
+import { generateService, type SdkSpec } from "./generator.ts";
+
+export interface GeneratorCliOptions {
+  /** Command description shown in --help. */
+  readonly description: string;
+  /** Absolute package root (usually `path.resolve(import.meta.dir, "..")`). */
+  readonly root: string;
+  /** Model directory default (relative to root). Default `.generated-specs`. */
+  readonly smithyDir?: string;
+  /** Output directory default (relative to root). Default `src/services`. */
+  readonly outDir?: string;
+  /** Model files to skip (e.g. a shared protocols model). */
+  readonly excludeModel?: (file: string) => boolean;
+  /** Directory of hand-authored models merged after the generated ones. */
+  readonly manualSpecsDir?: string;
+  /** RFC-6902 patch chain root. Default `patches`; `false` disables. */
+  readonly patchesDir?: string | false;
+  /**
+   * Model transform applied AFTER the patch chain, before generation —
+   * for whole-model rewrites that must see post-patch shape names (e.g.
+   * cloudflare's scope-twin structural dedup). May mutate the model;
+   * a returned log line is printed.
+   */
+  readonly transformModel?: (model: any, resource: string) => string | void;
+  /** The provider spec — built per model (metadata may vary per model). */
+  readonly spec: (model: any) => SdkSpec;
+}
+
+/** Run the generator CLI (BunRuntime main — call at module top level). */
+export const runGeneratorCli = (options: GeneratorCliOptions): void => {
+  const command = Command.make(
+    "generate",
+    {
+      smithy: Flag.string("smithy").pipe(
+        Flag.withDefault(options.smithyDir ?? ".generated-specs"),
+        Flag.withDescription("Directory of Smithy JSON models"),
+      ),
+      out: Flag.string("out").pipe(
+        Flag.withDefault(options.outDir ?? "src/services"),
+        Flag.withDescription("Output directory for generated service modules"),
+      ),
+      resource: Flag.string("resource").pipe(
+        Flag.withDefault(""),
+        Flag.withDescription("Only generate this resource (e.g. ai)"),
+      ),
+    },
+    (config) =>
+      Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const path = yield* Path.Path;
+        const root = options.root;
+        const smithyDir = path.resolve(root, config.smithy);
+        const outDir = path.resolve(root, config.out);
+
+        yield* Console.log("⚙️  generate");
+        yield* Console.log(`   Smithy: ${smithyDir}`);
+        yield* Console.log(`   Output: ${outDir}`);
+
+        // Generated models plus optional manual-specs (hand-authored models
+        // for APIs the provider's spec source doesn't cover). A manual model
+        // must not shadow a generated one.
+        const generated = (yield* fs.readDirectory(smithyDir))
+          .filter(
+            (f) => f.endsWith(".json") && !(options.excludeModel?.(f) ?? false),
+          )
+          .map((f) => ({ file: f, dir: smithyDir }));
+        const manualDir = options.manualSpecsDir
+          ? path.resolve(root, options.manualSpecsDir)
+          : undefined;
+        const manual =
+          manualDir && (yield* fs.exists(manualDir))
+            ? (yield* fs.readDirectory(manualDir))
+                .filter((f) => f.endsWith(".json"))
+                .map((f) => ({ file: f, dir: manualDir }))
+            : [];
+        for (const m of manual) {
+          if (generated.some((g) => g.file === m.file)) {
+            return yield* Effect.die(
+              new Error(
+                `${options.manualSpecsDir}/${m.file} shadows a generated model — rename or delete it`,
+              ),
+            );
+          }
+        }
+        const entries = [...generated, ...manual].sort((a, b) =>
+          a.file.localeCompare(b.file),
+        );
+
+        yield* fs.makeDirectory(outDir, { recursive: true });
+
+        const written: string[] = [];
+        let totalOps = 0;
+        let totalPatches = 0;
+        let staleOps = 0;
+        const badPatches: string[] = [];
+
+        // Orphan check: a patch directory matching no smithy model would be
+        // silently dropped — flag it instead.
+        const patchRoot =
+          options.patchesDir === false
+            ? undefined
+            : path.join(root, options.patchesDir ?? "patches");
+        if (patchRoot && (yield* fs.exists(patchRoot))) {
+          const resources = new Set(
+            entries.map((e) => e.file.replace(/\.json$/, "")),
+          );
+          for (const dir of yield* fs.readDirectory(patchRoot)) {
+            if (!resources.has(dir)) {
+              yield* Console.warn(
+                `⚠️  patches/${dir}/ matches no smithy model — orphaned?`,
+              );
+            }
+          }
+        }
+
+        for (const { file, dir } of entries) {
+          const resource = file.replace(/\.json$/, "");
+          if (config.resource && resource !== config.resource) continue;
+
+          const model = JSON.parse(
+            yield* fs.readFileString(path.join(dir, file)),
+          );
+
+          // Apply the RFC-6902 patch chain before generating. Hand-written
+          // *.manual.json patches apply after the generated ones — they
+          // usually target post-rename shape names.
+          const patchDir = patchRoot && path.join(patchRoot, resource);
+          if (patchDir && (yield* fs.exists(patchDir))) {
+            const patchFiles = (yield* fs.readDirectory(patchDir))
+              .filter((f) => f.endsWith(".json"))
+              .sort(
+                (a, b) =>
+                  Number(a.endsWith(".manual.json")) -
+                    Number(b.endsWith(".manual.json")) || a.localeCompare(b),
+              );
+            for (const pf of patchFiles) {
+              const parsed = JSON.parse(
+                yield* fs.readFileString(path.join(patchDir, pf)),
+              ) as PatchFile;
+              for (const patchOp of parsed.patches ?? []) {
+                try {
+                  applyOperation(model, patchOp);
+                } catch (e) {
+                  const msg = e instanceof Error ? e.message : String(e);
+                  if (isStaleTargetError(msg)) {
+                    staleOps++;
+                    yield* Console.warn(
+                      `   ⚠️  stale: ${resource}/${pf} [${patchOp.op} ${patchOp.path}]`,
+                    );
+                  } else {
+                    badPatches.push(
+                      `${resource}/${pf} [${patchOp.op} ${patchOp.path}]: ${msg}`,
+                    );
+                  }
+                }
+              }
+              totalPatches++;
+            }
+          }
+
+          if (options.transformModel) {
+            const note = options.transformModel(model, resource);
+            if (note) yield* Console.log(`   ${note}`);
+          }
+
+          const { code, operations } = generateService(
+            model,
+            options.spec(model),
+          );
+          if (operations === 0) continue;
+
+          yield* fs.writeFileString(path.join(outDir, `${resource}.ts`), code);
+          written.push(resource);
+          totalOps += operations;
+        }
+
+        if (badPatches.length) {
+          for (const b of badPatches) {
+            yield* Console.error(`❌ bad patch: ${b}`);
+          }
+          return yield* Effect.die(
+            new Error(
+              `${badPatches.length} malformed patch operation(s) — fix or remove them`,
+            ),
+          );
+        }
+
+        // Barrel — namespace per resource to avoid op-name collisions.
+        //
+        // A FULL run is authoritative: the barrel is exactly what was
+        // generated, so a resource whose model went away drops out of it.
+        //
+        // A `--resource` run only knows about the resource it generated, so
+        // rewriting the barrel from that would delete every other export —
+        // which is how #397 shipped a cloudflare barrel exporting `workers`
+        // and nothing else, silently dropping 119 services. Merge into the
+        // existing barrel instead, so `--resource` can still ADD a brand-new
+        // resource without removing anything.
+        const barrelPath = path.join(outDir, "index.ts");
+        const filtered = config.resource !== "";
+        let resources = written;
+        if (filtered && (yield* fs.exists(barrelPath))) {
+          const existing = (yield* fs.readFileString(barrelPath))
+            .split("\n")
+            .map((line) => /^export \* as (\S+) from/.exec(line)?.[1])
+            .filter((name): name is string => name !== undefined);
+          // Ordered exactly as a full run orders it: by MODEL FILENAME, so
+          // the `.json` takes part in the collation (`ai_gateway.json` sorts
+          // before `ai.json`, where the bare names sort the other way).
+          // Otherwise a filtered run reshuffles the barrel into diff noise.
+          resources = [...new Set([...existing, ...written])].sort((a, b) =>
+            `${a}.json`.localeCompare(`${b}.json`),
+          );
+        }
+
+        yield* fs.writeFileString(
+          barrelPath,
+          barrel(
+            `// AUTO-GENERATED by scripts/generate.ts. Do not edit.\n`,
+            resources.map((r) => ({ name: r, path: `./${r}.ts` })),
+          ),
+        );
+
+        yield* formatGenerated(outDir);
+
+        yield* Console.log(
+          `\n✅ Generated ${totalOps} operations across ${written.length} resource modules` +
+            (totalPatches
+              ? ` (${totalPatches} patch files applied${staleOps ? `, ${staleOps} stale op(s) skipped` : ""}).`
+              : "."),
+        );
+        yield* Console.log(`   ${path.join(outDir, "index.ts")}`);
+      }),
+  ).pipe(Command.withDescription(options.description));
+
+  BunRuntime.runMain(
+    Effect.provide(
+      Command.run(command, { version: "1.0.0" }),
+      BunServices.layer,
+    ),
+  );
+};

package/src/codegen/emit.ts

@@ -0,0 +1,207 @@
+/**
+ * Shared emission idioms for the SDK generators (dev-time only).
+ *
+ * Both SDK generators emit the same compile-time-performance pattern
+ * (ported from distilled PR #360):
+ *
+ *   export interface X { … }                      // hand-emitted type
+ *   export const X = S.suspend(() => S.Struct({…}))
+ *     .annotate({ identifier: "X" })
+ *     as any as S.Schema<X>;                      // no inference needed
+ *
+ * plus closed-alias string-union enums, `S.TaggedErrorClass` error classes, and
+ * `API.make(() => ({ … }))` operation consts. The helpers here own those
+ * shared skeletons; providers own the content strings (member pipes, trait
+ * calls, config fields). Emitted output is normalized by oxfmt afterwards,
+ * so helpers emit canonical token streams rather than matching historical
+ * whitespace.
+ */
+import { q } from "./naming.ts";
+
+/**
+ * The PURE annotation emitted before generated schema consts. A single
+ * `/*@__PURE__*​/` — Rolldown 1.1+ warns on the `/*#__PURE__*​/` form
+ * (distilled #374).
+ */
+export const PURE = "/*@__PURE__*/ ";
+
+/** `export interface X { … }` (or the empty-body form). */
+export const interfaceDecl = (name: string, fields: string[]): string =>
+  fields.length
+    ? `export interface ${name} {\n${fields.join("\n")}\n}`
+    : `export interface ${name} {}`;
+
+export interface SuspendConstOptions {
+  readonly name: string;
+  /** The schema expression inside the suspend thunk. */
+  readonly expr: string;
+  /** PURE marker(s), e.g. `"/*@__PURE__*​/ "`. Emitted verbatim before the expression. */
+  readonly pure?: string;
+  /** When set, `.annotate({ identifier: <name> })` is appended after the suspend. */
+  readonly annotateIdentifier?: boolean;
+  /** Extra annotation object source to use instead of the identifier default. */
+  readonly annotation?: string;
+  /**
+   * Explicit thunk return type (`(): S.Schema<X> =>`) — used for shapes in
+   * dependency cycles to stop circular type inference.
+   */
+  readonly thunkType?: string;
+  /** CF-style multiline body (`S.suspend(() =>\n<expr>,\n)`). */
+  readonly multiline?: boolean;
+  /** The cast target; defaults to `S.Schema<name>`. */
+  readonly castTo?: string;
+}
+
+/**
+ * The `export const X = S.suspend(…) … as any as S.Schema<X>;` skeleton
+ * shared by both generators.
+ */
+export const suspendConst = (o: SuspendConstOptions): string => {
+  const cast = o.castTo ?? `S.Schema<${o.name}>`;
+  const thunk = o.thunkType ? `(): ${o.thunkType} =>` : `() =>`;
+  const suspend = o.multiline
+    ? `S.suspend(${thunk}\n${o.expr},\n)`
+    : `S.suspend(${thunk} ${o.expr})`;
+  const annotate = o.annotation
+    ? `.annotate(${o.annotation})`
+    : o.annotateIdentifier
+      ? `.annotate({ identifier: ${q(o.name)} })`
+      : "";
+  return `export const ${o.name} = ${o.pure ?? ""}${suspend}${annotate} as any as ${cast};\n`;
+};
+
+/**
+ * Member-level lazy reference: `S.suspend(() => X).annotate({ identifier })`.
+ * The `typed` form adds an explicit `S.Schema<X>` thunk return type — used
+ * for references into dependency cycles to stop circular type inference.
+ */
+export const suspendRef = (name: string, typed = false): string =>
+  typed
+    ? `S.suspend((): S.Schema<${name}> => ${name}).annotate({ identifier: ${q(name)} })`
+    : `S.suspend(() => ${name}).annotate({ identifier: ${q(name)} })`;
+
+export interface EnumDeclOptions {
+  readonly name: string;
+  readonly values: readonly string[];
+  readonly pure?: string;
+  /** The schema const expression; both SDKs use `S.String` (open enums). */
+  readonly schemaExpr?: string;
+}
+
+/**
+ * String-union enum ALIAS: the spec's documented values as a CLOSED literal
+ * union (`type X = "a" | "b"`) — response readers match documented values
+ * exhaustively. INPUT references re-open the alias inline
+ * (`X | (string & {})`) so consumers can send tomorrow's values without an
+ * SDK update. The schema stays `S.String` in both directions (the
+ * protocols never validate enum membership, so undocumented wire values
+ * always pass through at runtime).
+ */
+export const enumDecl = (o: EnumDeclOptions): string[] => {
+  const union = o.values.length ? o.values.map(q).join(" | ") : "string";
+  return [
+    `export type ${o.name} = ${union};`,
+    `export const ${o.name} = ${o.pure ?? ""}${o.schemaExpr ?? "S.String"};\n`,
+  ];
+};
+
+export interface ErrorClassOptions {
+  readonly name: string;
+  /** The error tag; defaults to `name`. */
+  readonly tag?: string;
+  /** Field lines (`  key: S.String,`). */
+  readonly fields: readonly string[];
+  /** Optional extra argument(s) after the fields object (annotations). */
+  readonly annotations?: string;
+  /** `.pipe(…)` suffix (e.g. category decorators). */
+  readonly pipes?: string;
+  /** Wrap the class expression (e.g. `T.applyErrorMatchers(<cls>, …)`). */
+  readonly wrap?: (cls: string) => string;
+}
+
+/**
+ * `export class X extends /*@__PURE__*​/ S.TaggedErrorClass<X>()("X", { … }) {}`
+ *
+ * The PURE markers are what make an unused error class droppable. A class
+ * whose heritage clause is an unannotated call can never be tree-shaken —
+ * the bundler has to assume the call has side effects — so a consumer
+ * importing one operation would retain every error class in the module
+ * (distilled #191).
+ *
+ * `wrap` needs its own marker as well as the inner one: a pure call's
+ * ARGUMENTS are still evaluated, so annotating only
+ * `T.applyErrorMatchers(S.TaggedErrorClass…(…), […])` leaves the inner call
+ * holding the class alive. Verified against esbuild in both directions.
+ */
+export const errorClass = (o: ErrorClassOptions): string => {
+  const annotations = o.annotations ? `,\n${o.annotations}` : "";
+  const cls = `${PURE}S.TaggedErrorClass<${o.name}>()(${q(o.tag ?? o.name)}, {\n${o.fields.join("\n")}\n}${annotations})${o.pipes ?? ""}`;
+  const body = o.wrap ? `${PURE}${o.wrap(cls)}` : cls;
+  return `export class ${o.name} extends ${body} {}\n`;
+};
+
+export interface OperationConstOptions {
+  /** The exported (usually lowerFirst) operation name. */
+  readonly exportName: string;
+  /** Full type annotation (e.g. `API.OperationMethod<A, B, E, R>`). */
+  readonly typeAnnotation: string;
+  /** `API.make` or `API.makePaginated` (with namespace prefix). */
+  readonly factory: string;
+  /** The config object source (including braces). */
+  readonly config: string;
+  /** Optional extra factory argument (e.g. a pagination strategy). */
+  readonly extraArg?: string;
+  readonly pure?: string;
+  /**
+   * Widen the factory result to `any`, for annotations the factory's
+   * generic signature can't prove. The one case today: a paginated
+   * operation's `items` element type comes from the pagination trait's
+   * `items` PATH — a runtime string — so the factory can only infer the
+   * structural fallback while the annotation names the real element type.
+   *
+   * Unlike the schema consts' `as any as S.Schema<X>`, the target doesn't
+   * need restating here: the const carries its own annotation, which IS
+   * the assignment target, so a bare `as any` lands in the same place.
+   * Restating it would double every paginated operation's declaration —
+   * ~40k lines across the SDKs — for no added checking.
+   */
+  readonly castToAnnotation?: boolean;
+}
+
+/** `export const op: T = API.make(() => ({ … }));` */
+export const operationConst = (o: OperationConstOptions): string =>
+  `export const ${o.exportName}: ${o.typeAnnotation} = ${o.pure ?? ""}${o.factory}(() => (${o.config})${
+    o.extraArg ? `, ${o.extraArg}` : ""
+  })${o.castToAnnotation ? ` as any` : ""};\n`;
+
+import { tsKey } from "./naming.ts";
+
+export interface InterfaceFieldOptions {
+  readonly name: string;
+  readonly type: string;
+  readonly optional: boolean;
+  readonly doc?: string;
+}
+
+/** Interface field line(s): optional doc comment + `  name?: Type;`. */
+export const interfaceField = (o: InterfaceFieldOptions): string[] => [
+  ...(o.doc ? [`  /** ${o.doc} */`] : []),
+  `  ${tsKey(o.name)}${o.optional ? "?" : ""}: ${o.type};`,
+];
+
+/** `export type <Op>Error = A | B | <CommonErrors>;` */
+export const errorUnionAlias = (
+  opName: string,
+  errorNames: readonly string[],
+  commonRef: string,
+): string =>
+  `export type ${opName}Error = ${[...errorNames, commonRef].join(" | ")};`;
+
+/** Namespaced barrel: `export * as name from "./file.ts";` per entry. */
+export const barrel = (
+  header: string,
+  entries: ReadonlyArray<{ name: string; path: string }>,
+): string =>
+  header +
+  entries.map((e) => `export * as ${e.name} from ${q(e.path)};`).join("\n") +
+  "\n";

package/src/codegen/format.ts

@@ -0,0 +1,47 @@
+/**
+ * Post-generation formatting (dev-time only).
+ *
+ * The emitters produce a canonical token stream, not formatted source, so
+ * every generator formats what it wrote before finishing. Without it a
+ * generate run leaves the whole output directory dirty against the committed
+ * (formatted) files, and a real regression is indistinguishable from
+ * whitespace in the diff.
+ *
+ * Lives here so the shared {@link runGeneratorCli} and the providers with
+ * their own pipelines run the identical step.
+ */
+import { Console, Effect } from "effect";
+
+/** Run a dev-time tool, failing the generate run if it does. */
+export const runTool = (
+  argv: readonly string[],
+): Effect.Effect<void, never, never> =>
+  Effect.tryPromise({
+    try: () =>
+      Bun.spawn([...argv], { stdout: "inherit", stderr: "inherit" }).exited,
+    catch: (cause) => new Error(`${argv[0]} failed to start: ${cause}`),
+  }).pipe(
+    Effect.flatMap((code) =>
+      code === 0
+        ? Effect.void
+        : Effect.die(new Error(`${argv.join(" ")} exited with ${code}`)),
+    ),
+    Effect.catchCause((cause) => Effect.die(cause)),
+  );
+
+/** Format a generated directory in place. */
+export const formatGenerated = (dir: string) =>
+  Effect.flatMap(Console.log(`\n🧹 Formatting ${dir}`), () =>
+    runTool(["bunx", "oxfmt", dir]),
+  );
+
+/**
+ * Lint-fix then format. `oxlint --fix` can leave its rewrites unformatted,
+ * so the formatter has to run after it, not before.
+ */
+export const lintAndFormatGenerated = (dir: string) =>
+  Effect.flatMap(Console.log(`\n🧹 Linting and formatting ${dir}`), () =>
+    Effect.flatMap(runTool(["bunx", "oxlint", "--fix", dir]), () =>
+      runTool(["bunx", "oxfmt", dir]),
+    ),
+  );