@distilled.cloud/core 0.30.3 → 1.0.0-rc.10

package/src/codegen/operations.ts

@@ -0,0 +1,76 @@
+/**
+ * Operation-level smithy helpers for the SDK generators (dev-time only).
+ */
+import { local } from "./naming.ts";
+import type { ShapeMap } from "./graph.ts";
+
+export interface OpEntry {
+  id: string;
+  def: any;
+}
+
+/** All operation shapes, sorted by local name. */
+export const collectOperations = (shapes: ShapeMap): OpEntry[] => {
+  const operations: OpEntry[] = [];
+  for (const [id, def] of Object.entries(shapes)) {
+    if (def.type === "operation") operations.push({ id, def });
+  }
+  operations.sort((a, b) => local(a.id).localeCompare(local(b.id)));
+  return operations;
+};
+
+/** The model's shape namespace (from the first operation, else any shape). */
+export const modelNamespace = (
+  operations: readonly OpEntry[],
+  shapes: ShapeMap,
+  fallback: string,
+): string =>
+  operations.length
+    ? operations[0].id.split("#")[0]
+    : (Object.keys(shapes)[0]?.split("#")[0] ?? fallback);
+
+/**
+ * Resolve an operation's input/output targets, synthesizing an empty named
+ * structure for `smithy.api#Unit` so every operation has a shape that can
+ * carry operation-level traits (mutates `shapes`).
+ */
+export const ensureNamedIo = (
+  shapes: ShapeMap,
+  op: OpEntry,
+  ns: string,
+): { input: string; output: string } => {
+  const opName = local(op.id);
+  let input = op.def.input?.target ?? "smithy.api#Unit";
+  let output = op.def.output?.target ?? "smithy.api#Unit";
+  if (input === "smithy.api#Unit") {
+    input = `${ns}#${opName}Request`;
+    shapes[input] = { type: "structure", members: {} };
+  }
+  if (output === "smithy.api#Unit") {
+    output = `${ns}#${opName}Response`;
+    shapes[output] = { type: "structure", members: {} };
+  }
+  return { input, output };
+};
+
+/**
+ * Every error shape id referenced by the operations' `errors` lists —
+ * deduplicated, existing in the model, sorted by local name.
+ */
+export const collectOpErrorIds = (
+  operations: readonly OpEntry[],
+  shapes: ShapeMap,
+): string[] => {
+  const errorIds: string[] = [];
+  const seen = new Set<string>();
+  for (const op of operations) {
+    for (const e of op.def.errors ?? []) {
+      if (!seen.has(e.target) && shapes[e.target]) {
+        seen.add(e.target);
+        errorIds.push(e.target);
+      }
+    }
+  }
+  errorIds.sort((a, b) => local(a).localeCompare(local(b)));
+  return errorIds;
+};

package/src/codegen/pagination.ts

@@ -0,0 +1,71 @@
+/**
+ * Shared pagination-trait handling for the SDK generators (dev-time only).
+ *
+ * Both SDKs source `smithy.api#paginated`; AWS merges operation-level
+ * config over service-level defaults, cloudflare validates the trait
+ * against the operation's actual members (degrading to a plain op when it
+ * can't work). Both behaviors live here.
+ */
+import type { PaginatedTrait } from "../pagination.ts";
+
+/**
+ * Merge an operation-level paginated trait over service-level defaults
+ * (Smithy semantics: operations may specify partial pagination and inherit
+ * the rest from the service).
+ */
+export const mergePaginated = (
+  op: Partial<PaginatedTrait> | undefined,
+  service: Partial<PaginatedTrait> | undefined,
+): PaginatedTrait | undefined => {
+  if (!op) return undefined;
+  const merged = {
+    mode: op.mode ?? service?.mode,
+    inputToken: op.inputToken ?? service?.inputToken,
+    outputToken: op.outputToken ?? service?.outputToken,
+    items: op.items ?? service?.items,
+    pageSize: op.pageSize ?? service?.pageSize,
+  };
+  return Object.fromEntries(
+    Object.entries(merged).filter(([, v]) => v !== undefined),
+  ) as PaginatedTrait;
+};
+
+export interface ValidatePaginatedOptions {
+  readonly trait: Partial<PaginatedTrait>;
+  /** TS-facing member names of the operation input shape. */
+  readonly inputNames: ReadonlySet<string>;
+  /** TS-facing member names of the operation output shape. */
+  readonly outputNames: ReadonlySet<string>;
+  /** Items path fallback when the trait omits `items` (e.g. `"result"`). */
+  readonly itemsFallback?: string;
+  /** Output names accepted even when not modeled (e.g. `"resultInfo"`). */
+  readonly syntheticOutputs?: ReadonlySet<string>;
+}
+
+export interface ValidatedPagination {
+  readonly ok: boolean;
+  /** Root member of the items path (`"result.items"` → `"result"`). */
+  readonly itemsRoot: string;
+}
+
+/**
+ * A paginated op must actually carry its page/cursor token on the input
+ * and its items member on the output — otherwise `.pages()` would loop or
+ * yield nothing. Returns the items-path root for emitters that promote it
+ * to required.
+ */
+export const validatePaginated = (
+  o: ValidatePaginatedOptions,
+): ValidatedPagination => {
+  const itemsRoot = String(o.trait.items ?? o.itemsFallback ?? "").split(
+    ".",
+  )[0];
+  const tokenOk =
+    o.trait.mode === "single" ||
+    (typeof o.trait.inputToken === "string" &&
+      o.inputNames.has(o.trait.inputToken));
+  const itemsOk =
+    o.outputNames.has(itemsRoot) ||
+    (o.syntheticOutputs?.has(itemsRoot) ?? false);
+  return { ok: tokenOk && itemsOk, itemsRoot };
+};

package/src/codegen/patches.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  FINALIZED_KEY,
+  danglingTargets,
+  finalizeConvert,
+  syncServiceOperations,
+} from "./patches.ts";
+
+const model = () => ({
+  smithy: "2.0",
+  shapes: {
+    "ns#Svc": {
+      type: "service",
+      operations: [{ target: "ns#AppsList" }, { target: "ns#AppsGet" }],
+    },
+    "ns#AppsList": {
+      type: "operation",
+      input: { target: "ns#AppsListRequest" },
+      output: { target: "smithy.api#Unit" },
+    },
+    "ns#AppsListRequest": { type: "structure", members: {} },
+    "ns#AppsGet": {
+      type: "operation",
+      input: { target: "smithy.api#Unit" },
+      output: { target: "smithy.api#Unit" },
+    },
+  },
+});
+
+const scaffold = (patches?: Record<string, unknown>) => {
+  const root = mkdtempSync(join(tmpdir(), "finalize-"));
+  mkdirSync(join(root, ".generated-specs"));
+  writeFileSync(
+    join(root, ".generated-specs", "svc.json"),
+    JSON.stringify(model()),
+  );
+  if (patches) {
+    mkdirSync(join(root, "patches", "svc"), { recursive: true });
+    writeFileSync(
+      join(root, "patches", "svc", "a.json"),
+      JSON.stringify(patches),
+    );
+  }
+  return root;
+};
+
+const read = (root: string) =>
+  JSON.parse(readFileSync(join(root, ".generated-specs", "svc.json"), "utf8"));
+
+describe("finalizeConvert", () => {
+  test("renames, syncs the service list, stamps the marker", async () => {
+    const root = scaffold();
+    await finalizeConvert({ root });
+    const m = read(root);
+    expect(Object.keys(m.shapes).sort()).toEqual(
+      ["ns#Svc", "ns#ListApps", "ns#ListAppsRequest", "ns#GetApp"].sort(),
+    );
+    expect(m.shapes["ns#Svc"].operations).toEqual([
+      { target: "ns#ListApps" },
+      { target: "ns#GetApp" },
+    ]);
+    expect(m.metadata[FINALIZED_KEY]).toBe(true);
+  });
+
+  test("refuses a second pass", async () => {
+    const root = scaffold();
+    await finalizeConvert({ root });
+    await expect(finalizeConvert({ root })).rejects.toThrow(
+      /already finalized/,
+    );
+  });
+
+  test("applies Smithy patches and repairs the service list after a move", async () => {
+    const root = scaffold({
+      patches: [
+        { op: "move", from: "/shapes/ns#AppsGet", path: "/shapes/ns#FetchApp" },
+      ],
+    });
+    await finalizeConvert({ root });
+    const m = read(root);
+    expect(m.shapes["ns#FetchApp"]).toBeDefined();
+    expect(m.shapes["ns#Svc"].operations.map((o: any) => o.target)).toEqual([
+      "ns#ListApps",
+      "ns#FetchApp",
+    ]);
+  });
+
+  test("fails on a stale patch pointer by default", async () => {
+    const root = scaffold({
+      patches: [{ op: "remove", path: "/shapes/ns#Missing" }],
+    });
+    await expect(finalizeConvert({ root })).rejects.toThrow(/failed/);
+  });
+
+  test("fails on dangling targets and leaves the model unstamped", async () => {
+    const root = scaffold({
+      patches: [
+        {
+          op: "replace",
+          path: "/shapes/ns#AppsList/output/target",
+          value: "ns#Nope",
+        },
+      ],
+    });
+    await expect(finalizeConvert({ root })).rejects.toThrow(/do not exist/);
+    expect(read(root).metadata?.[FINALIZED_KEY]).toBeUndefined();
+  });
+});
+
+describe("danglingTargets", () => {
+  test("ignores prelude ids", () => {
+    expect(danglingTargets(model())).toEqual([]);
+  });
+});
+
+describe("syncServiceOperations", () => {
+  test("drops missing and appends unlisted, keeping order", () => {
+    const m = model();
+    delete (m.shapes as any)["ns#AppsGet"];
+    (m.shapes as any)["ns#Zed"] = { type: "operation" };
+    expect(syncServiceOperations(m)).toBe(1);
+    expect((m.shapes as any)["ns#Svc"].operations).toEqual([
+      { target: "ns#AppsList" },
+      { target: "ns#Zed" },
+    ]);
+  });
+});

package/src/codegen/patches.ts

@@ -0,0 +1,291 @@
+/**
+ * RFC-6902 patch application for convert (dev-time only).
+ *
+ * OpenAPI ops (`/paths`, `/components`, …) apply to the spec before
+ * conversion; Smithy ops (`/shapes`, `/metadata`, `/smithy`) apply to the
+ * model after conversion. `.generated-specs` is the patched model.
+ * `scripts/generate.ts` does not apply patches.
+ */
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import {
+  applyOperation,
+  isStaleTargetError,
+  type JsonPatchOperation,
+  type PatchFile,
+} from "../json-patch.ts";
+import { verbNounSmithyModel } from "./rewrite-operation-ids.ts";
+
+export type OnStalePatch = "fail" | "warn";
+
+export interface ApplyPatchesResult {
+  files: number;
+  applied: number;
+  stale: number;
+  errors: string[];
+}
+
+/** Smithy-model JSON pointers — OpenAPI has no `/shapes` tree. */
+export const isSmithyPatchPath = (pointer: string): boolean =>
+  pointer.startsWith("/shapes") ||
+  pointer.startsWith("/metadata") ||
+  pointer.startsWith("/smithy");
+
+const exists = async (p: string): Promise<boolean> => {
+  try {
+    await fs.access(p);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * RFC-6902 files in `dir`: every `*.json`, `*.manual.json` last (those
+ * usually target post-rename shape names). Missing dir → `[]`.
+ */
+export const listRfc6902PatchFiles = async (dir: string): Promise<string[]> => {
+  if (!(await exists(dir))) return [];
+  return (await fs.readdir(dir))
+    .filter((f) => f.endsWith(".json"))
+    .sort(
+      (a, b) =>
+        Number(a.endsWith(".manual.json")) -
+          Number(b.endsWith(".manual.json")) || a.localeCompare(b),
+    )
+    .map((f) => path.join(dir, f));
+};
+
+export const applyRfc6902Files = async (
+  target: unknown,
+  files: readonly string[],
+  opts: {
+    readonly onStalePatch?: OnStalePatch;
+    readonly include?: (op: JsonPatchOperation) => boolean;
+    readonly label?: (file: string) => string;
+  } = {},
+): Promise<ApplyPatchesResult> => {
+  const onStalePatch = opts.onStalePatch ?? "fail";
+  const include = opts.include ?? (() => true);
+  const result: ApplyPatchesResult = {
+    files: 0,
+    applied: 0,
+    stale: 0,
+    errors: [],
+  };
+  for (const file of files) {
+    const parsed = JSON.parse(await fs.readFile(file, "utf8")) as PatchFile;
+    const label = opts.label?.(file) ?? path.basename(file);
+    result.files++;
+    for (const patchOp of parsed.patches ?? []) {
+      if (!include(patchOp)) continue;
+      try {
+        applyOperation(target, patchOp);
+        result.applied++;
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        const line = `${label} [${patchOp.op} ${patchOp.path}]`;
+        if (isStaleTargetError(e)) {
+          result.stale++;
+          if (onStalePatch === "fail") {
+            result.errors.push(`${line}: stale target (${msg})`);
+          } else {
+            console.warn(`   ⚠️  stale: ${line}`);
+          }
+        } else {
+          result.errors.push(`${line}: ${msg}`);
+        }
+      }
+    }
+  }
+  return result;
+};
+
+/**
+ * An RFC-6902 `move` that renames an operation shape leaves the service's
+ * `operations` list pointing at the old id. Drop entries whose target no
+ * longer exists and append operation shapes the list is missing, keeping
+ * the existing order. Returns the number of services whose list changed.
+ */
+export const syncServiceOperations = (model: {
+  shapes?: Record<string, any>;
+}): number => {
+  const shapes = model.shapes ?? {};
+  const opIds = Object.keys(shapes).filter(
+    (id) => shapes[id]?.type === "operation",
+  );
+  let changed = 0;
+  for (const def of Object.values(shapes)) {
+    if (def?.type !== "service") continue;
+    const current: Array<{ target: string }> = def.operations ?? [];
+    const kept = current.filter((o) => shapes[o.target]?.type === "operation");
+    const listed = new Set(kept.map((o) => o.target));
+    const after = [
+      ...kept,
+      ...opIds.filter((id) => !listed.has(id)).map((target) => ({ target })),
+    ];
+    if (JSON.stringify(current) !== JSON.stringify(after)) {
+      def.operations = after;
+      changed++;
+    }
+  }
+  return changed;
+};
+
+/**
+ * Every `target` in a Smithy model must resolve to a shape or a prelude
+ * (`smithy.api#…`) id. Returns the dangling ones as `"<owner> → <target>"`.
+ * A convert that produces any is broken — generate would emit references
+ * to types that do not exist.
+ */
+export const danglingTargets = (model: {
+  shapes?: Record<string, any>;
+}): string[] => {
+  const shapes = model.shapes ?? {};
+  const ids = new Set(Object.keys(shapes));
+  const out: string[] = [];
+  const walk = (owner: string, node: unknown): void => {
+    if (Array.isArray(node)) {
+      for (const item of node) walk(owner, item);
+      return;
+    }
+    if (node === null || typeof node !== "object") return;
+    for (const [key, value] of Object.entries(
+      node as Record<string, unknown>,
+    )) {
+      if (key === "target" && typeof value === "string") {
+        if (!value.startsWith("smithy.") && !ids.has(value)) {
+          out.push(`${owner} → ${value}`);
+        }
+      } else {
+        walk(owner, value);
+      }
+    }
+  };
+  for (const [id, def] of Object.entries(shapes)) walk(id, def);
+  return out;
+};
+
+/**
+ * Last step of every convert: Smithy RFC-6902 patches, then verbNoun
+ * operation names, then an optional model transform, then a reference
+ * check. Writes models back so `.generated-specs` is what generate
+ * compiles. `outDir` may be nested (GCP `stable/` / `unstable/`).
+ *
+ * Input must be FRESHLY converted models. Smithy patches are `move`/`add`
+ * ops that are not idempotent, and a `transform` may not be either, so
+ * running this over a model that already went through it is an error —
+ * re-run the package's `convert` instead.
+ */
+export const finalizeConvert = async (o: {
+  readonly root: string;
+  readonly outDir?: string;
+  readonly patchesDir?: string | false;
+  readonly exclude?: (file: string) => boolean;
+  readonly include?: (resource: string) => boolean;
+  readonly transform?: (model: any, resource: string) => string | void;
+  readonly operationNaming?: "as-is" | "verbNoun";
+  readonly onStalePatch?: OnStalePatch;
+}): Promise<void> => {
+  const specsDir = path.resolve(o.root, o.outDir ?? ".generated-specs");
+  if (!(await exists(specsDir))) return;
+  const patchesDir =
+    o.patchesDir === false
+      ? undefined
+      : path.resolve(o.root, o.patchesDir ?? "patches");
+  const naming = o.operationNaming ?? "verbNoun";
+
+  const walk = async (dir: string): Promise<string[]> => {
+    const out: string[] = [];
+    for (const ent of await fs.readdir(dir, { withFileTypes: true })) {
+      const p = path.join(dir, ent.name);
+      if (ent.isDirectory()) {
+        out.push(...(await walk(p)));
+      } else if (
+        ent.name.endsWith(".json") &&
+        !(o.exclude?.(ent.name) ?? false)
+      ) {
+        out.push(p);
+      }
+    }
+    return out;
+  };
+
+  const files = (await walk(specsDir)).sort((a, b) => a.localeCompare(b));
+  const broken: string[] = [];
+  for (const modelPath of files) {
+    const resource = path.basename(modelPath, ".json");
+    if (o.include && !o.include(resource)) continue;
+    const model = JSON.parse(await fs.readFile(modelPath, "utf8"));
+    if (model.metadata?.[FINALIZED_KEY]) {
+      throw new Error(
+        `${path.relative(o.root, modelPath)} was already finalized — finalizeConvert is not idempotent; re-run this package's convert from the spec instead`,
+      );
+    }
+    if (patchesDir) {
+      const patchFiles = await listRfc6902PatchFiles(
+        path.join(patchesDir, resource),
+      );
+      const applied = await applyRfc6902Files(model, patchFiles, {
+        onStalePatch: o.onStalePatch,
+        include: (op) => isSmithyPatchPath(op.path),
+        label: (f) => `${resource}/${path.basename(f)}`,
+      });
+      if (applied.errors.length) {
+        for (const err of applied.errors) console.error(`❌ bad patch: ${err}`);
+        throw new Error(
+          `${applied.errors.length} patch operation(s) failed for ${resource} — fix the pointers or delete the patch`,
+        );
+      }
+      if (applied.applied > 0) {
+        console.log(
+          `   patched ${resource}: ${applied.files} file(s), ${applied.applied} op(s)` +
+            (applied.stale ? `, ${applied.stale} stale` : ""),
+        );
+      }
+    }
+
+    if (naming === "verbNoun") {
+      const { renamed, collisions } = verbNounSmithyModel(model);
+      if (renamed > 0) {
+        console.log(`   verbNoun ${resource}: renamed ${renamed} operation(s)`);
+      }
+      for (const c of collisions) {
+        console.warn(
+          `   ⚠️  verbNoun collision ${resource}: ${c} (kept original)`,
+        );
+      }
+    }
+
+    const note = o.transform?.(model, resource);
+    if (note) console.log(`   ${note}`);
+
+    syncServiceOperations(model);
+
+    const dangling = danglingTargets(model);
+    if (dangling.length) {
+      broken.push(`${resource}: ${dangling.length} dangling target(s)`);
+      for (const d of dangling.slice(0, 5)) console.error(`   ❌ ${d}`);
+      if (dangling.length > 5) {
+        console.error(`   … ${dangling.length - 5} more`);
+      }
+      // Leave the unfinalized model on disk for inspection; never stamp
+      // a model that generate cannot compile.
+      continue;
+    }
+
+    model.metadata = { ...model.metadata, [FINALIZED_KEY]: true };
+    await fs.writeFile(modelPath, `${JSON.stringify(model, null, 2)}\n`);
+  }
+  if (broken.length) {
+    throw new Error(
+      `finalizeConvert: ${broken.length} model(s) reference shapes that do not exist:\n  ${broken.join("\n  ")}`,
+    );
+  }
+};
+
+/**
+ * Metadata marker stamped by {@link finalizeConvert}. Guards against a
+ * second pass over the same file (see the note on finalizeConvert).
+ */
+export const FINALIZED_KEY = "distilled.finalized";

package/src/codegen/prelude.ts

@@ -0,0 +1,70 @@
+/**
+ * Smithy prelude scalar baselines + reference resolvers for the SDK
+ * generators (dev-time only).
+ *
+ * The JSON flavor maps timestamps/blobs to plain strings — the baseline for
+ * SDKs whose wire format is JSON-over-docs (cloudflare). Generators with
+ * richer models (AWS) emit real timestamp/blob schemas themselves and only
+ * fall back here for untyped scalars.
+ */
+import { isPrelude, local } from "./naming.ts";
+import { isForwardRef } from "./graph.ts";
+
+/** Prelude shape → schema expression (JSON-wire flavor). */
+export const JSON_PRELUDE: Record<string, string> = {
+  String: "S.String",
+  Boolean: "S.Boolean",
+  Double: "S.Number",
+  Float: "S.Number",
+  Integer: "S.Number",
+  Long: "S.Number",
+  BigInteger: "S.Number",
+  BigDecimal: "S.Number",
+  Timestamp: "S.String",
+  Blob: "S.String",
+  Document: "S.Unknown",
+  Unit: "S.Struct({})",
+};
+
+/** TypeScript type for each prelude shape, mirroring {@link JSON_PRELUDE}. */
+export const TS_JSON_PRELUDE: Record<string, string> = {
+  String: "string",
+  Boolean: "boolean",
+  Double: "number",
+  Float: "number",
+  Integer: "number",
+  Long: "number",
+  BigInteger: "number",
+  BigDecimal: "number",
+  Timestamp: "string",
+  Blob: "string",
+  Document: "unknown",
+  Unit: "{}",
+};
+
+/**
+ * Schema reference resolver: prelude shapes map through the prelude table,
+ * forward references (including into cycles) wrap in `S.suspend`.
+ */
+export const makeSchemaRef =
+  (
+    prelude: Record<string, string>,
+    indexOf: ReadonlyMap<string, number>,
+  ): ((target: string, selfIdx: number) => string) =>
+  (target, selfIdx) => {
+    if (isPrelude(target)) return prelude[local(target)] ?? "S.Unknown";
+    const name = local(target);
+    if (isForwardRef(indexOf, target, selfIdx)) {
+      return `S.suspend(() => ${name})`;
+    }
+    return name;
+  };
+
+/**
+ * TypeScript type reference resolver. Named shapes all get an emitted
+ * interface / type alias, so forward references are always fine.
+ */
+export const makeTsRef =
+  (tsPrelude: Record<string, string>): ((target: string) => string) =>
+  (target) =>
+    isPrelude(target) ? (tsPrelude[local(target)] ?? "unknown") : local(target);