@distilled.cloud/core 1.0.0-rc.7 → 1.0.0-rc.9

package/src/codegen/openapi-cli.ts

@@ -2,29 +2,29 @@
  * OpenAPI → Smithy pipeline helper (dev-time only, provider-agnostic).
  *
  * Owns the spec-side pipeline every OpenAPI-sourced provider shares: read the
- * spec file, apply the RFC-6902 patch chain to the OpenAPI document (v0
- * semantics: `*.patch.json` files in sorted name order, `{ description,
- * patches }` shape; stale targets warn+skip, malformed patches fail the run),
- * run the optional preprocess hook, convert with
- * {@link convertOpenApiToSmithy}, and write `.generated-specs/<name>.json`.
+ * spec file, apply OpenAPI RFC-6902 ops to the document, convert with
+ * {@link convertOpenApiToSmithy} (which owns verbNoun naming), apply Smithy
+ * RFC-6902 ops (`/shapes`, `/metadata`) to the model, and write
+ * `.generated-specs/<name>.json`. Stale targets fail unless
+ * `onStalePatch: "warn"`.
  *
- * A provider's `scripts/convert.ts` is: a `runOpenApiConvert` call. The
- * separate `scripts/generate.ts` (an SdkSpec + `runGeneratorCli`) then
- * compiles the written models. Note: providers whose OpenAPI patches live in
- * `patches/` should pass `patchesDir: false` to `runGeneratorCli` — the
- * patch chain applies HERE, to the OpenAPI document, not to the Smithy model.
+ * A provider's `scripts/convert.ts` is: a `runOpenApiConvert` call.
+ * `scripts/generate.ts` compiles the already-patched models and does not
+ * apply patches.
  */
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
-import {
-  applyOperation,
-  isStaleTargetError,
-  type PatchFile,
-} from "../json-patch.ts";
 import {
   convertOpenApiToSmithy,
   type OpenApiConvertOptions,
 } from "./openapi.ts";
+import {
+  applyRfc6902Files,
+  finalizeConvert,
+  isSmithyPatchPath,
+  listRfc6902PatchFiles,
+} from "./patches.ts";
+import { resolveSpecPath } from "./spec-path.ts";
 
 export interface OpenApiSpecEntry {
   /** Output model name — written to `<outDir>/<name>.json`. */
@@ -35,7 +35,7 @@ export interface OpenApiSpecEntry {
    * Hook between patching and conversion (e.g. path prefixing, server
    * rewrites). May mutate the spec in place or return a replacement.
    */
-  readonly preprocess?: (spec: any) => unknown | void;
+  readonly preprocess?: (spec: any) => unknown | void | Promise<unknown | void>;
   /** Per-spec converter option overrides (merged over the shared options). */
   readonly options?: Partial<OpenApiConvertOptions>;
 }
@@ -46,9 +46,10 @@ export interface RunOpenApiConvertOptions {
   readonly specs: readonly OpenApiSpecEntry[];
   /**
    * RFC-6902 patch chain root, relative to `root`. Default `"patches"`;
-   * `false` disables. Layout: `<patchesDir>/<name>/*.patch.json` when the
+   * `false` disables. Layout: `<patchesDir>/<name>/*.json` when the
    * per-spec directory exists; for single-spec providers a flat
-   * `<patchesDir>/*.patch.json` also works.
+   * `<patchesDir>/*.json` also works. Ops under `/shapes` or `/metadata`
+   * apply to the Smithy model after conversion; the rest apply to OpenAPI.
    */
   readonly patchesDir?: string | false;
   /** Output directory, relative to `root`. Default `".generated-specs"`. */
@@ -60,6 +61,19 @@ export interface RunOpenApiConvertOptions {
   readonly parse?: (text: string, specPath: string) => unknown;
   /** Shared converter options (per-spec `options` merge over these). */
   readonly options: OpenApiConvertOptions;
+  /**
+   * What to do when a patch JSON pointer does not resolve. Default `"fail"`:
+   * silent skip is how a whole patch chain can vanish after an upstream path
+   * prefix change. `"warn"` restores the old skip-and-continue behaviour.
+   */
+  readonly onStalePatch?: "fail" | "warn";
+  /**
+   * Run {@link finalizeConvert} on the written models (reference check +
+   * finalized marker; naming already happened in the converter). Default
+   * true. Pass false when the caller finalizes once after several convert
+   * steps (Fly machines + sprites + addons).
+   */
+  readonly finalize?: boolean;
 }
 
 const exists = async (p: string): Promise<boolean> => {
@@ -81,6 +95,7 @@ export const runOpenApiConvert = async (
       ? undefined
       : path.resolve(o.root, o.patchesDir ?? "patches");
   const parse = o.parse ?? ((text: string) => JSON.parse(text));
+  const onStalePatch = o.onStalePatch ?? "fail";
 
   await fs.mkdir(outDir, { recursive: true });
 
@@ -88,10 +103,11 @@ export const runOpenApiConvert = async (
   console.log(`   Output: ${outDir}`);
 
   for (const entry of o.specs) {
-    const specPath = path.resolve(o.root, entry.specPath);
+    // Production path by default; `specs/.local` under DISTILLED_SPECS_LOCAL.
+    const specPath = resolveSpecPath(o.root, entry.specPath);
     let spec: any = parse(await fs.readFile(specPath, "utf8"), specPath);
 
-    // ---- RFC-6902 patch chain (applies to the OpenAPI document) ----
+    // ---- RFC-6902: OpenAPI ops before convert, Smithy ops after ----
     let patchDir: string | undefined;
     if (patchRoot && (await exists(patchRoot))) {
       const perSpec = path.join(patchRoot, entry.name);
@@ -101,66 +117,66 @@ export const runOpenApiConvert = async (
         patchDir = patchRoot;
       }
     }
-    let patchFileCount = 0;
-    let staleOps = 0;
-    const badPatches: string[] = [];
-    if (patchDir) {
-      const files = (await fs.readdir(patchDir))
-        .filter((f) => f.endsWith(".patch.json"))
-        .sort((a, b) => a.localeCompare(b));
-      for (const pf of files) {
-        const parsed = JSON.parse(
-          await fs.readFile(path.join(patchDir, pf), "utf8"),
-        ) as PatchFile;
-        for (const patchOp of parsed.patches ?? []) {
-          try {
-            applyOperation(spec, patchOp);
-          } catch (e) {
-            const msg = e instanceof Error ? e.message : String(e);
-            if (isStaleTargetError(msg)) {
-              // Spec drift — the patch target no longer exists upstream.
-              staleOps++;
-              console.warn(
-                `   ⚠️  stale: ${entry.name}/${pf} [${patchOp.op} ${patchOp.path}]`,
-              );
-            } else {
-              badPatches.push(
-                `${entry.name}/${pf} [${patchOp.op} ${patchOp.path}]: ${msg}`,
-              );
-            }
-          }
-        }
-        patchFileCount++;
-      }
-    }
-    if (badPatches.length) {
-      for (const b of badPatches) console.error(`❌ bad patch: ${b}`);
+    const patchFiles = patchDir ? await listRfc6902PatchFiles(patchDir) : [];
+    const patchLabel = (file: string) => `${entry.name}/${path.basename(file)}`;
+    const openapiPatches = await applyRfc6902Files(spec, patchFiles, {
+      onStalePatch,
+      include: (op) => !isSmithyPatchPath(op.path),
+      label: patchLabel,
+    });
+    if (openapiPatches.errors.length) {
+      for (const b of openapiPatches.errors)
+        console.error(`❌ bad patch: ${b}`);
       throw new Error(
-        `${badPatches.length} malformed patch operation(s) — fix or remove them`,
+        `${openapiPatches.errors.length} patch operation(s) failed — fix the pointers or delete the patch`,
       );
     }
 
     // ---- Preprocess hook ----
     if (entry.preprocess) {
-      const replaced = entry.preprocess(spec);
+      const replaced = await entry.preprocess(spec);
       if (replaced !== undefined) spec = replaced;
     }
 
-    // ---- Convert + write ----
+    // ---- Convert, Smithy patches, write ----
     const model = convertOpenApiToSmithy(spec, {
       ...o.options,
       ...entry.options,
     });
+    const smithyPatches = await applyRfc6902Files(model, patchFiles, {
+      onStalePatch,
+      include: (op) => isSmithyPatchPath(op.path),
+      label: patchLabel,
+    });
+    if (smithyPatches.errors.length) {
+      for (const b of smithyPatches.errors) console.error(`❌ bad patch: ${b}`);
+      throw new Error(
+        `${smithyPatches.errors.length} Smithy patch operation(s) failed — fix the pointers or delete the patch`,
+      );
+    }
     const opCount = Object.values(model.shapes).filter(
       (s: any) => s.type === "operation",
     ).length;
     const outPath = path.join(outDir, `${entry.name}.json`);
     await fs.writeFile(outPath, JSON.stringify(model, null, 2) + "\n");
+    const staleOps = openapiPatches.stale + smithyPatches.stale;
     console.log(
       `   ✅ ${entry.name}: ${opCount} operations, ${Object.keys(model.shapes).length} shapes` +
-        (patchFileCount
-          ? ` (${patchFileCount} patch file(s) applied${staleOps ? `, ${staleOps} stale op(s) skipped` : ""})`
+        (patchFiles.length
+          ? ` (${patchFiles.length} patch file(s) applied${staleOps ? `, ${staleOps} stale op(s) skipped` : ""})`
           : ""),
     );
   }
+
+  if (o.finalize !== false) {
+    const written = new Set(o.specs.map((s) => s.name));
+    await finalizeConvert({
+      root: o.root,
+      outDir,
+      // Patches and naming already ran above.
+      patchesDir: false,
+      operationNaming: "as-is",
+      include: (resource) => written.has(resource),
+    });
+  }
 };

package/src/codegen/openapi.ts

@@ -9,7 +9,9 @@
  *
  *   • one operation per (path × get/post/put/patch/delete — plus head/options
  *     when a provider opts in via `extraHttpMethods`), deprecated skipped by
- *     default; op shape name = PascalCase(operationId)
+ *     default; op shape name = PascalCase(operationId), or verbNoun
+ *     (`Apps_list` → `ListApps`) when {@link OpenApiConvertOptions.operationNaming}
+ *     is `"verbNoun"`
  *   • input `<Op>Request`: path params → `smithy.api#httpLabel` (+required),
  *     query params → `smithy.api#httpQuery`, header params →
  *     `smithy.api#httpHeader` when `headerParams` is on (dropped otherwise,
@@ -42,6 +44,15 @@
  *     (with the non-standard `mode` member the core runtime dispatches on)
  */
 
+import {
+  isMechanicalOperationId,
+  isVerbatimRouteId,
+  pathToVerbNoun,
+  resolveOperationName,
+  toVerbNoun,
+  type OperationIdRewrite,
+} from "./rewrite-operation-ids.ts";
+
 // ============================================================================
 // Trait ids (the `com.distilled.openapi` vocabulary)
 // ============================================================================
@@ -177,6 +188,29 @@ export interface OpenApiConvertOptions {
    * {@link ERROR_MATCHERS_TRAIT} traits; overrides are merged over it.
    */
   readonly errorShapes?: Readonly<Record<string, any>>;
+  /**
+   * How to turn an OpenAPI `operationId` into the Smithy/SDK operation name.
+   * This is a convert policy, not a spec patch — the OpenAPI document is
+   * left alone.
+   *
+   * - `"verbNoun"` (default): `Apps_list` / `ConfigsList` / `showContact` →
+   *   `listApps` / `listConfigs` / `getApp`. Already verb-first ids stay.
+   *   An operation with no `operationId`, or a mechanical one that only
+   *   restates the method and path (`get-api-card`, `post_v1_users`), is
+   *   named from the route instead: `GET /users` → `listUsers`,
+   *   `GET /users/{id}` → `getUser`, `POST /users/{id}/reset` →
+   *   `resetUser`.
+   * - `"as-is"`: `PascalCase(operationId)` (`Apps_list` → `AppsList`).
+   *
+   * {@link operationNames} overrides win (lookup by `"METHOD path"`, then
+   * operationId).
+   */
+  readonly operationNaming?: "as-is" | "verbNoun";
+  /**
+   * Per-operation name overrides for {@link operationNaming}. Use
+   * `"METHOD path"` keys when two methods share an `operationId`.
+   */
+  readonly operationNames?: OperationIdRewrite;
 }
 
 export interface SmithyModel {
@@ -1079,6 +1113,37 @@ const successSchema = (
   return { schema: undefined };
 };
 
+/**
+ * Whether the success body is a collection: an array, or an object whose
+ * array members outnumber its scalar ones (envelopes like `{ data: [] }`,
+ * `{ items: [], total }`). `undefined` when there is no body to judge.
+ */
+const responseIsCollection = (
+  ctx: Ctx,
+  responses: any,
+  order: readonly string[],
+): boolean | undefined => {
+  const { schema } = successSchema(ctx, responses, order);
+  if (!schema) return undefined;
+  const s = deref(ctx, schema);
+  if (!s || typeof s !== "object") return undefined;
+  if (s.type === "array" || s.items) return true;
+  const props = s.properties;
+  if (!props || typeof props !== "object") {
+    return s.allOf || s.oneOf || s.anyOf ? undefined : false;
+  }
+  let arrays = 0;
+  let others = 0;
+  for (const prop of Object.values(props)) {
+    const p = deref(ctx, prop);
+    if (p?.type === "array" || p?.items) arrays++;
+    else others++;
+  }
+  if (arrays === 0) return false;
+  // `{ data: [...] }`, `{ items, next_cursor }`, `{ results, count, page }`
+  return arrays >= 1 && others <= 3;
+};
+
 // ============================================================================
 // Pagination detection (v0 detectPagination)
 // ============================================================================
@@ -1227,11 +1292,60 @@ export const convertOpenApiToSmithy = (
       if (!op || typeof op !== "object") continue;
       if (skipDeprecated && op.deprecated === true) continue;
 
-      const opName = pascal(
+      const rawOperationId =
         typeof op.operationId === "string" && op.operationId
           ? op.operationId
-          : `${method}_${rawPath}`,
-      );
+          : `${method}_${rawPath}`;
+      const idCtx = { path: rawPath, method };
+      const naming = options.operationNaming ?? "verbNoun";
+      let named =
+        options.operationNames !== undefined
+          ? resolveOperationName(options.operationNames, rawOperationId, idCtx)
+          : undefined;
+      if (named === undefined && naming === "verbNoun") {
+        // A mechanical id (`get-api-card`, or none at all) is named from
+        // the route. A hand-chosen one that merely starts with the method
+        // and reads the route in order (`get-feeds` on /feeds) keeps its
+        // own nouns; only the HTTP-method verb is normalised.
+        if (typeof op.operationId !== "string" || !op.operationId) {
+          // No id: everything comes from the route, including whether a
+          // GET on a collection route is `list` or `get`.
+          named = pathToVerbNoun(idCtx, {
+            returnsCollection: responseIsCollection(
+              ctx,
+              op.responses,
+              successStatuses,
+            ),
+          });
+        } else if (isMechanicalOperationId(op.operationId, idCtx)) {
+          // Method-prefixed and reading the route (`get-feeds`,
+          // `postV1AppsByAppIdPromote`, `deleteProjectJWKS`): keep the
+          // author's tokens and casing; normalise `post`/`patch`, drop
+          // parameter clauses (`ByAppId`, `ById`) and api/version roots.
+          named = pathToVerbNoun(idCtx, {
+            nouns: op.operationId,
+            verbatim: isVerbatimRouteId(op.operationId, idCtx),
+          });
+        } else {
+          named = toVerbNoun(rawOperationId);
+        }
+      }
+      let resolved: string = named ?? rawOperationId;
+      // Two routes can derive the same name (`/collections/{slug}` and
+      // `/collections/{slug}-{id}`); suffix the trailing parameter rather
+      // than a counter.
+      if (
+        naming === "verbNoun" &&
+        ctx.names.has(pascal(resolved)) &&
+        isMechanicalOperationId(
+          typeof op.operationId === "string" ? op.operationId : undefined,
+          idCtx,
+        )
+      ) {
+        const lastParam = /\{([^}]+)\}[^/]*$/.exec(rawPath)?.[1];
+        if (lastParam) resolved = `${resolved}By${pascal(lastParam)}`;
+      }
+      const opName = pascal(resolved);
 
       const params = collectParams(ctx, pathItem, op);
 

package/src/codegen/patches.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  FINALIZED_KEY,
+  danglingTargets,
+  finalizeConvert,
+  syncServiceOperations,
+} from "./patches.ts";
+
+const model = () => ({
+  smithy: "2.0",
+  shapes: {
+    "ns#Svc": {
+      type: "service",
+      operations: [{ target: "ns#AppsList" }, { target: "ns#AppsGet" }],
+    },
+    "ns#AppsList": {
+      type: "operation",
+      input: { target: "ns#AppsListRequest" },
+      output: { target: "smithy.api#Unit" },
+    },
+    "ns#AppsListRequest": { type: "structure", members: {} },
+    "ns#AppsGet": {
+      type: "operation",
+      input: { target: "smithy.api#Unit" },
+      output: { target: "smithy.api#Unit" },
+    },
+  },
+});
+
+const scaffold = (patches?: Record<string, unknown>) => {
+  const root = mkdtempSync(join(tmpdir(), "finalize-"));
+  mkdirSync(join(root, ".generated-specs"));
+  writeFileSync(
+    join(root, ".generated-specs", "svc.json"),
+    JSON.stringify(model()),
+  );
+  if (patches) {
+    mkdirSync(join(root, "patches", "svc"), { recursive: true });
+    writeFileSync(
+      join(root, "patches", "svc", "a.json"),
+      JSON.stringify(patches),
+    );
+  }
+  return root;
+};
+
+const read = (root: string) =>
+  JSON.parse(readFileSync(join(root, ".generated-specs", "svc.json"), "utf8"));
+
+describe("finalizeConvert", () => {
+  test("renames, syncs the service list, stamps the marker", async () => {
+    const root = scaffold();
+    await finalizeConvert({ root });
+    const m = read(root);
+    expect(Object.keys(m.shapes).sort()).toEqual(
+      ["ns#Svc", "ns#ListApps", "ns#ListAppsRequest", "ns#GetApp"].sort(),
+    );
+    expect(m.shapes["ns#Svc"].operations).toEqual([
+      { target: "ns#ListApps" },
+      { target: "ns#GetApp" },
+    ]);
+    expect(m.metadata[FINALIZED_KEY]).toBe(true);
+  });
+
+  test("refuses a second pass", async () => {
+    const root = scaffold();
+    await finalizeConvert({ root });
+    await expect(finalizeConvert({ root })).rejects.toThrow(
+      /already finalized/,
+    );
+  });
+
+  test("applies Smithy patches and repairs the service list after a move", async () => {
+    const root = scaffold({
+      patches: [
+        { op: "move", from: "/shapes/ns#AppsGet", path: "/shapes/ns#FetchApp" },
+      ],
+    });
+    await finalizeConvert({ root });
+    const m = read(root);
+    expect(m.shapes["ns#FetchApp"]).toBeDefined();
+    expect(m.shapes["ns#Svc"].operations.map((o: any) => o.target)).toEqual([
+      "ns#ListApps",
+      "ns#FetchApp",
+    ]);
+  });
+
+  test("fails on a stale patch pointer by default", async () => {
+    const root = scaffold({
+      patches: [{ op: "remove", path: "/shapes/ns#Missing" }],
+    });
+    await expect(finalizeConvert({ root })).rejects.toThrow(/failed/);
+  });
+
+  test("fails on dangling targets and leaves the model unstamped", async () => {
+    const root = scaffold({
+      patches: [
+        {
+          op: "replace",
+          path: "/shapes/ns#AppsList/output/target",
+          value: "ns#Nope",
+        },
+      ],
+    });
+    await expect(finalizeConvert({ root })).rejects.toThrow(/do not exist/);
+    expect(read(root).metadata?.[FINALIZED_KEY]).toBeUndefined();
+  });
+});
+
+describe("danglingTargets", () => {
+  test("ignores prelude ids", () => {
+    expect(danglingTargets(model())).toEqual([]);
+  });
+});
+
+describe("syncServiceOperations", () => {
+  test("drops missing and appends unlisted, keeping order", () => {
+    const m = model();
+    delete (m.shapes as any)["ns#AppsGet"];
+    (m.shapes as any)["ns#Zed"] = { type: "operation" };
+    expect(syncServiceOperations(m)).toBe(1);
+    expect((m.shapes as any)["ns#Svc"].operations).toEqual([
+      { target: "ns#AppsList" },
+      { target: "ns#Zed" },
+    ]);
+  });
+});