@distilled.cloud/core 0.30.2 → 1.0.0-rc.1

package/src/codegen/graph.ts

@@ -0,0 +1,151 @@
+/**
+ * Shared shape-graph utilities for the SDK generators (dev-time only).
+ *
+ * Works over plain Smithy-JSON shape maps (`{ "ns#Name": { type, members,
+ * … } }`). Providers supply the shape map and (optionally) a custom
+ * dependency extractor; reachability, topological ordering, and cycle
+ * analysis are generic.
+ */
+import { isPrelude } from "./naming.ts";
+
+export type ShapeMap = Record<string, any>;
+
+/**
+ * Direct dependencies of a shape: structure/union member targets, list
+ * member, map key/value. Operations/services are not traversed here —
+ * providers seed reachability from operation I/O explicitly.
+ */
+export const shapeDeps = (def: any): string[] => {
+  if (!def) return [];
+  if (def.type === "structure" || def.type === "union") {
+    return Object.values(def.members ?? {}).map((m: any) => m.target);
+  }
+  if (def.type === "list") return [def.member.target];
+  if (def.type === "map") {
+    return def.key ? [def.key.target, def.value.target] : [def.value.target];
+  }
+  return [];
+};
+
+/**
+ * Every shape reachable from `roots` through `deps`, excluding prelude
+ * shapes and ids missing from the map.
+ */
+export const reachableFrom = (
+  shapes: ShapeMap,
+  roots: Iterable<string>,
+  deps: (def: any) => string[] = shapeDeps,
+): Set<string> => {
+  const reachable = new Set<string>();
+  const visit = (id: string) => {
+    if (isPrelude(id) || reachable.has(id) || !shapes[id]) return;
+    reachable.add(id);
+    for (const dep of deps(shapes[id])) visit(dep);
+  };
+  for (const r of roots) visit(r);
+  return reachable;
+};
+
+/**
+ * Dependencies-first topological order over `ids`. Cycles are tolerated:
+ * a back-edge is simply skipped, so mutually-recursive shapes come out in
+ * discovery order and emitters break the cycle with `S.suspend` at
+ * forward references (see {@link forwardRef}).
+ */
+export const topoOrder = (
+  shapes: ShapeMap,
+  ids: Iterable<string>,
+  deps: (def: any) => string[] = shapeDeps,
+): string[] => {
+  const order: string[] = [];
+  const done = new Set<string>();
+  const stack = new Set<string>();
+  const walk = (id: string) => {
+    if (done.has(id) || stack.has(id) || isPrelude(id) || !shapes[id]) return;
+    stack.add(id);
+    for (const dep of deps(shapes[id])) walk(dep);
+    stack.delete(id);
+    done.add(id);
+    order.push(id);
+  };
+  for (const id of ids) walk(id);
+  return order;
+};
+
+/** Position index for {@link forwardRef} decisions. */
+export const orderIndex = (order: readonly string[]): Map<string, number> => {
+  const indexOf = new Map<string, number>();
+  order.forEach((id, i) => indexOf.set(id, i));
+  return indexOf;
+};
+
+/**
+ * Whether a reference from the shape at `selfIdx` to `target` points
+ * forward in emission order (including into a cycle) and therefore needs
+ * `S.suspend(() => …)` to defer evaluation.
+ */
+export const isForwardRef = (
+  indexOf: ReadonlyMap<string, number>,
+  target: string,
+  selfIdx: number,
+): boolean => {
+  const ti = indexOf.get(target);
+  return ti !== undefined && ti > selfIdx;
+};
+
+/**
+ * All shape ids that participate in a dependency cycle (Tarjan SCC —
+ * components of size > 1, plus direct self-loops). Used by generators
+ * that must know cyclicity independent of emission position (e.g. to add
+ * an explicit `S.Schema<T>` return type on suspended thunks and break
+ * circular type inference).
+ */
+export const cyclicShapeIds = (
+  shapes: ShapeMap,
+  ids: Iterable<string>,
+  deps: (def: any) => string[] = shapeDeps,
+): Set<string> => {
+  const idSet = new Set(ids);
+  const index = new Map<string, number>();
+  const low = new Map<string, number>();
+  const onStack = new Set<string>();
+  const stack: string[] = [];
+  const cyclic = new Set<string>();
+  let counter = 0;
+
+  const strongConnect = (v: string) => {
+    index.set(v, counter);
+    low.set(v, counter);
+    counter++;
+    stack.push(v);
+    onStack.add(v);
+
+    for (const w of deps(shapes[v])) {
+      if (!idSet.has(w) || !shapes[w]) continue;
+      if (!index.has(w)) {
+        strongConnect(w);
+        low.set(v, Math.min(low.get(v)!, low.get(w)!));
+      } else if (onStack.has(w)) {
+        low.set(v, Math.min(low.get(v)!, index.get(w)!));
+      }
+    }
+
+    if (low.get(v) === index.get(v)) {
+      const component: string[] = [];
+      let w: string;
+      do {
+        w = stack.pop()!;
+        onStack.delete(w);
+        component.push(w);
+      } while (w !== v);
+      if (component.length > 1 || deps(shapes[v]).some((d) => d === v)) {
+        for (const c of component) cyclic.add(c);
+      }
+    }
+  };
+
+  for (const id of idSet) {
+    if (!index.has(id) && shapes[id]) strongConnect(id);
+  }
+  return cyclic;
+};

package/src/codegen/members.ts

@@ -0,0 +1,71 @@
+/**
+ * Structure-member analysis for the SDK generators (dev-time only).
+ *
+ * The TS-facing name policy is the provider's (cloudflare camelCases wire
+ * names, AWS keeps them) — supplied as `nameFn`. Wire-name resolution from
+ * the smithy HTTP binding traits is generic smithy semantics.
+ */
+import { oneLine } from "./naming.ts";
+
+export interface MemberBase {
+  /** Original smithy member name. */
+  readonly name: string;
+  /** TS-facing name (per `nameFn`, deduplicated within the structure). */
+  readonly tsName: string;
+  readonly target: string;
+  readonly required: boolean;
+  readonly doc: string | undefined;
+  readonly traits: Record<string, any>;
+}
+
+/**
+ * Base analysis of a structure's members: TS naming (with collision
+ * dedup — a clashing name falls back to the original, then a numeric
+ * suffix), required-ness, and one-line docs. Binding classification and
+ * anything provider-specific layer on top via `traits`.
+ */
+export const memberBases = (
+  def: any,
+  nameFn: (memberName: string) => string = (n) => n,
+): MemberBase[] => {
+  const used = new Set<string>();
+  return Object.entries(def.members ?? {}).map(([mn, m]: [string, any]) => {
+    const traits = m.traits ?? {};
+    let tsName = nameFn(mn);
+    if (used.has(tsName)) tsName = mn;
+    let k = 2;
+    while (used.has(tsName)) tsName = `${nameFn(mn)}${k++}`;
+    used.add(tsName);
+    return {
+      name: mn,
+      tsName,
+      target: m.target,
+      required: "smithy.api#required" in traits,
+      doc: oneLine(traits["smithy.api#documentation"]),
+      traits,
+    };
+  });
+};
+
+/**
+ * The wire name a bound member serializes under, per the smithy HTTP
+ * binding traits: labels use the member name (URI placeholders), query and
+ * header use the trait value when it's a non-empty string, everything else
+ * uses `jsonName` when present.
+ */
+export const smithyWireName = (
+  traits: Record<string, any>,
+  memberName: string,
+  binding: "label" | "query" | "header" | "other",
+): string => {
+  if (binding === "label") return memberName;
+  if (binding === "query") {
+    const v = traits["smithy.api#httpQuery"];
+    return typeof v === "string" && v ? v : memberName;
+  }
+  if (binding === "header") {
+    const v = traits["smithy.api#httpHeader"];
+    return typeof v === "string" && v ? v : memberName;
+  }
+  return traits["smithy.api#jsonName"] ?? memberName;
+};

package/src/codegen/naming.ts

@@ -0,0 +1,86 @@
+/**
+ * Shared naming utilities for the SDK generators (dev-time only — imported by
+ * `packages/<sdk>/scripts/generate.ts`, never by runtime code).
+ */
+
+/** JSON-string-quote a value for emission into generated source. */
+export const q = (s: string): string => JSON.stringify(s);
+
+/** Smithy shape id → local name (`ns#Shape` → `Shape`). */
+export const local = (id: string): string => id.split("#")[1] ?? id;
+
+/** Whether a shape id is a smithy prelude shape (`smithy.api#…`). */
+export const isPrelude = (id: string): boolean => id.startsWith("smithy.api#");
+
+const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
+
+/** Emit a TS object key: bare when a valid identifier, quoted otherwise. */
+export const tsKey = (s: string): string => (IDENT.test(s) ? s : q(s));
+
+/** snake_case / kebab-case wire name → camelCase TS-facing name. */
+export const camel = (s: string): string =>
+  s.replace(/[_-]+([A-Za-z0-9])/g, (_, c: string) => c.toUpperCase());
+
+/** Reserved words that can't be `const` names. */
+export const RESERVED = new Set([
+  "break",
+  "case",
+  "catch",
+  "class",
+  "const",
+  "continue",
+  "debugger",
+  "default",
+  "delete",
+  "do",
+  "else",
+  "enum",
+  "export",
+  "extends",
+  "false",
+  "finally",
+  "for",
+  "function",
+  "if",
+  "import",
+  "in",
+  "instanceof",
+  "let",
+  "new",
+  "null",
+  "package",
+  "return",
+  "static",
+  "super",
+  "switch",
+  "this",
+  "throw",
+  "true",
+  "try",
+  "typeof",
+  "var",
+  "void",
+  "while",
+  "with",
+  "yield",
+]);
+
+/**
+ * Lower-case the first character for an exported operation name — unless the
+ * result is a reserved word, in which case the original casing is kept.
+ */
+export const lowerFirst = (s: string): string => {
+  const lowered = s.charAt(0).toLowerCase() + s.slice(1);
+  return RESERVED.has(lowered) ? s : lowered;
+};
+
+/** Upper-case the first character. */
+export const upperFirst = (s: string): string =>
+  s.charAt(0).toUpperCase() + s.slice(1);
+
+/**
+ * Collapse a smithy documentation string to a single safe line for a
+ * `/** … *​/` comment.
+ */
+export const oneLine = (s: string | undefined): string | undefined =>
+  s ? s.replace(/\s+/g, " ").replace(/\*\//g, "*\\/").trim() : undefined;

package/src/codegen/openapi-cli.ts

@@ -0,0 +1,166 @@
+/**
+ * OpenAPI → Smithy pipeline helper (dev-time only, provider-agnostic).
+ *
+ * Owns the spec-side pipeline every OpenAPI-sourced provider shares: read the
+ * spec file, apply the RFC-6902 patch chain to the OpenAPI document (v0
+ * semantics: `*.patch.json` files in sorted name order, `{ description,
+ * patches }` shape; stale targets warn+skip, malformed patches fail the run),
+ * run the optional preprocess hook, convert with
+ * {@link convertOpenApiToSmithy}, and write `.generated-specs/<name>.json`.
+ *
+ * A provider's `scripts/convert.ts` is: a `runOpenApiConvert` call. The
+ * separate `scripts/generate.ts` (an SdkSpec + `runGeneratorCli`) then
+ * compiles the written models. Note: providers whose OpenAPI patches live in
+ * `patches/` should pass `patchesDir: false` to `runGeneratorCli` — the
+ * patch chain applies HERE, to the OpenAPI document, not to the Smithy model.
+ */
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import {
+  applyOperation,
+  isStaleTargetError,
+  type PatchFile,
+} from "../json-patch.ts";
+import {
+  convertOpenApiToSmithy,
+  type OpenApiConvertOptions,
+} from "./openapi.ts";
+
+export interface OpenApiSpecEntry {
+  /** Output model name — written to `<outDir>/<name>.json`. */
+  readonly name: string;
+  /** Spec file path, relative to `root` (or absolute). */
+  readonly specPath: string;
+  /**
+   * Hook between patching and conversion (e.g. path prefixing, server
+   * rewrites). May mutate the spec in place or return a replacement.
+   */
+  readonly preprocess?: (spec: any) => unknown | void;
+  /** Per-spec converter option overrides (merged over the shared options). */
+  readonly options?: Partial<OpenApiConvertOptions>;
+}
+
+export interface RunOpenApiConvertOptions {
+  /** Absolute package root (usually `path.resolve(import.meta.dir, "..")`). */
+  readonly root: string;
+  readonly specs: readonly OpenApiSpecEntry[];
+  /**
+   * RFC-6902 patch chain root, relative to `root`. Default `"patches"`;
+   * `false` disables. Layout: `<patchesDir>/<name>/*.patch.json` when the
+   * per-spec directory exists; for single-spec providers a flat
+   * `<patchesDir>/*.patch.json` also works.
+   */
+  readonly patchesDir?: string | false;
+  /** Output directory, relative to `root`. Default `".generated-specs"`. */
+  readonly outDir?: string;
+  /**
+   * Spec text parser. Default `JSON.parse` — the seam for YAML specs
+   * (`parse: (text) => YAML.parse(text)`).
+   */
+  readonly parse?: (text: string, specPath: string) => unknown;
+  /** Shared converter options (per-spec `options` merge over these). */
+  readonly options: OpenApiConvertOptions;
+}
+
+const exists = async (p: string): Promise<boolean> => {
+  try {
+    await fs.access(p);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/** Run the OpenAPI→Smithy conversion pipeline (plain async — call from a script). */
+export const runOpenApiConvert = async (
+  o: RunOpenApiConvertOptions,
+): Promise<void> => {
+  const outDir = path.resolve(o.root, o.outDir ?? ".generated-specs");
+  const patchRoot =
+    o.patchesDir === false
+      ? undefined
+      : path.resolve(o.root, o.patchesDir ?? "patches");
+  const parse = o.parse ?? ((text: string) => JSON.parse(text));
+
+  await fs.mkdir(outDir, { recursive: true });
+
+  console.log("🛠️  openapi → smithy");
+  console.log(`   Output: ${outDir}`);
+
+  for (const entry of o.specs) {
+    const specPath = path.resolve(o.root, entry.specPath);
+    let spec: any = parse(await fs.readFile(specPath, "utf8"), specPath);
+
+    // ---- RFC-6902 patch chain (applies to the OpenAPI document) ----
+    let patchDir: string | undefined;
+    if (patchRoot && (await exists(patchRoot))) {
+      const perSpec = path.join(patchRoot, entry.name);
+      if (await exists(perSpec)) {
+        patchDir = perSpec;
+      } else if (o.specs.length === 1) {
+        patchDir = patchRoot;
+      }
+    }
+    let patchFileCount = 0;
+    let staleOps = 0;
+    const badPatches: string[] = [];
+    if (patchDir) {
+      const files = (await fs.readdir(patchDir))
+        .filter((f) => f.endsWith(".patch.json"))
+        .sort((a, b) => a.localeCompare(b));
+      for (const pf of files) {
+        const parsed = JSON.parse(
+          await fs.readFile(path.join(patchDir, pf), "utf8"),
+        ) as PatchFile;
+        for (const patchOp of parsed.patches ?? []) {
+          try {
+            applyOperation(spec, patchOp);
+          } catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            if (isStaleTargetError(msg)) {
+              // Spec drift — the patch target no longer exists upstream.
+              staleOps++;
+              console.warn(
+                `   ⚠️  stale: ${entry.name}/${pf} [${patchOp.op} ${patchOp.path}]`,
+              );
+            } else {
+              badPatches.push(
+                `${entry.name}/${pf} [${patchOp.op} ${patchOp.path}]: ${msg}`,
+              );
+            }
+          }
+        }
+        patchFileCount++;
+      }
+    }
+    if (badPatches.length) {
+      for (const b of badPatches) console.error(`❌ bad patch: ${b}`);
+      throw new Error(
+        `${badPatches.length} malformed patch operation(s) — fix or remove them`,
+      );
+    }
+
+    // ---- Preprocess hook ----
+    if (entry.preprocess) {
+      const replaced = entry.preprocess(spec);
+      if (replaced !== undefined) spec = replaced;
+    }
+
+    // ---- Convert + write ----
+    const model = convertOpenApiToSmithy(spec, {
+      ...o.options,
+      ...entry.options,
+    });
+    const opCount = Object.values(model.shapes).filter(
+      (s: any) => s.type === "operation",
+    ).length;
+    const outPath = path.join(outDir, `${entry.name}.json`);
+    await fs.writeFile(outPath, JSON.stringify(model, null, 2) + "\n");
+    console.log(
+      `   ✅ ${entry.name}: ${opCount} operations, ${Object.keys(model.shapes).length} shapes` +
+        (patchFileCount
+          ? ` (${patchFileCount} patch file(s) applied${staleOps ? `, ${staleOps} stale op(s) skipped` : ""})`
+          : ""),
+    );
+  }
+};