@essential-apps/shopify-test-runner 1.0.0

package/src/scripts/captureContracts.ts

@@ -0,0 +1,675 @@
+#!/usr/bin/env node
+/**
+ * captureContracts — bootstrap step for the operation-contract
+ * conformance system. See docs/CONTRACTS.md for the full design.
+ *
+ * What it does:
+ *   1. Walk every `.ts/.tsx/.js/.jsx` file under `--glob` (default
+ *      `./app`).
+ *   2. Extract every `#graphql` template literal.
+ *   3. Execute each operation against an IN-PROCESS offline mock
+ *      (Admin GraphQL or Storefront GraphQL, per `--api`).
+ *   4. Write one JSON-per-operation under `tests/test-offline/contracts/<api>/`.
+ *
+ * The captured response is `capturedFrom: "offline"` — a tentative
+ * bootstrap. The conformance suite's `verify-contracts-from-live`
+ * probe later re-captures from real Shopify; on each successful
+ * live-capture the contract is overwritten with `capturedFrom: "live"`
+ * — the authoritative ground truth.
+ *
+ * Variables: an operation with required variables (`ID!`, `String!`,
+ * etc.) needs sample values to execute. We auto-synthesise sensible
+ * defaults for primitive types (`ID!` → first seeded product GID;
+ * `String!` → `"sample"`; `Int!` → `1`; `Boolean!` → `true`). Complex
+ * input objects are NOT synthesised — the operation is captured
+ * without execution and `response: null` + `error: "needs fixtures"`
+ * is recorded. A consuming app can hand-write `tests/test-offline/contracts/
+ * fixtures.json` mapping operation names to variables to cover those.
+ *
+ * Output is idempotent — same input + same offline mock = same
+ * contracts. Commit them.
+ */
+import {
+  readdirSync,
+  readFileSync,
+  statSync,
+  writeFileSync,
+  mkdirSync,
+  existsSync,
+} from 'node:fs';
+import { extname, resolve, dirname, basename, relative } from 'node:path';
+import {
+  parse as parseGraphql,
+  validate as validateGraphql,
+  buildSchema,
+  typeFromAST,
+  isNonNullType,
+  isListType,
+  isScalarType,
+  isEnumType,
+  isInputObjectType,
+  TypeInfo,
+  visit,
+  visitWithTypeInfo,
+  Kind,
+  type GraphQLSchema,
+  type DocumentNode,
+  type OperationDefinitionNode,
+  type TypeNode,
+  type GraphQLType,
+} from 'graphql';
+import {
+  loadAdminSdl,
+  loadStorefrontSdl,
+  createAdminApi,
+  createStorefrontApi,
+} from '@essential-apps/shopify-test-shopify-api';
+import { ShopState } from '@essential-apps/shopify-test-storefront';
+
+type ApiType = 'admin' | 'storefront';
+
+interface Args {
+  api: ApiType;
+  glob: string;
+  outDir: string;
+  fixturesPath: string;
+  cwd: string;
+  quiet: boolean;
+}
+
+function parseArgs(): Args {
+  const argv = process.argv.slice(2);
+  const out: Args = {
+    api: 'admin',
+    glob: './app',
+    outDir: '',
+    fixturesPath: '',
+    cwd: process.cwd(),
+    quiet: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i] ?? '';
+    if (a === '--api' && i + 1 < argv.length) {
+      const next = argv[++i] ?? '';
+      if (next !== 'admin' && next !== 'storefront') {
+        console.error(`--api must be "admin" or "storefront"`);
+        process.exit(2);
+      }
+      out.api = next;
+    } else if (a === '--glob' && i + 1 < argv.length) {
+      out.glob = argv[++i] ?? out.glob;
+    } else if (a === '--out' && i + 1 < argv.length) {
+      out.outDir = argv[++i] ?? '';
+    } else if (a === '--fixtures' && i + 1 < argv.length) {
+      out.fixturesPath = argv[++i] ?? '';
+    } else if (a === '--quiet') {
+      out.quiet = true;
+    } else {
+      console.error(`unknown arg: ${a}`);
+      process.exit(2);
+    }
+  }
+  // Default output dir scopes contracts under the consuming app's
+  // tests/test-offline/contracts/<api>/ — co-located with the spec files.
+  if (!out.outDir) {
+    out.outDir = resolve(out.cwd, 'tests/test-offline/contracts', out.api);
+  } else {
+    out.outDir = resolve(out.cwd, out.outDir);
+  }
+  if (!out.fixturesPath) {
+    out.fixturesPath = resolve(out.cwd, 'tests/test-offline/contracts/fixtures.json');
+  }
+  return out;
+}
+
+/**
+ * Walk a directory tree, yielding paths whose extension is a JS/TS
+ * source file. Skips `node_modules`, `dist`, `.git`, `build` — the
+ * usual non-source noise.
+ */
+function* walkSources(root: string): Generator<string> {
+  let entries: string[];
+  try {
+    entries = readdirSync(root);
+  } catch {
+    return;
+  }
+  for (const name of entries) {
+    if (
+      name === 'node_modules' ||
+      name === 'dist' ||
+      name === '.git' ||
+      name === 'build'
+    )
+      continue;
+    const full = resolve(root, name);
+    let st;
+    try {
+      st = statSync(full);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) {
+      yield* walkSources(full);
+    } else if (st.isFile()) {
+      const ext = extname(name);
+      if (ext === '.ts' || ext === '.tsx' || ext === '.js' || ext === '.jsx') {
+        yield full;
+      }
+    }
+  }
+}
+
+interface ExtractedOp {
+  file: string;
+  line: number;
+  source: string;
+  /** Operation name extracted from the document (or '__anon_<line>__'). */
+  name: string;
+}
+
+/**
+ * Extract every `#graphql ...` template literal from a file. Parses
+ * each to discover its operation name (so contracts get filenames
+ * that match `getShop` rather than `__anon_42__`).
+ */
+function extractOperations(file: string, content: string): ExtractedOp[] {
+  const ops: ExtractedOp[] = [];
+  const regex = /`\s*#graphql\b([^`]*)`/g;
+  let m: RegExpExecArray | null;
+  while ((m = regex.exec(content)) !== null) {
+    const line = content.slice(0, m.index).split('\n').length;
+    const source = m[1] ?? '';
+    const name = deriveOperationName(source, file, line);
+    ops.push({ file, line, source, name });
+  }
+  return ops;
+}
+
+/**
+ * Pick a stable, filesystem-safe slug for an operation. Preference
+ * order:
+ *
+ *   1. The operation's declared name —
+ *      `query getShop { ... }` → `getShop`.
+ *   2. A type+field slug derived from the first top-level selection
+ *      — `query { shop { name } }` → `anon_query_shop`. Stable across
+ *      file renames (only the first selected field is in the slug).
+ *   3. Filename+line fallback when the document is unparseable.
+ *
+ * Why anon slugs matter: contracts are committed to the consuming
+ * app; the slug is the filename. Anon ops whose slug depends on
+ * the file path produce noisy diffs whenever the call site moves.
+ * Type+field slugs survive refactors as long as the operation
+ * keeps its first selection.
+ */
+function deriveOperationName(source: string, file: string, line: number): string {
+  const fileBase =
+    file.split('/').pop()?.replace(/[^a-zA-Z0-9_-]/g, '_') ?? 'unknown';
+  const fallback = `__anon_${fileBase}_L${line}__`;
+  let doc: DocumentNode;
+  try {
+    doc = parseGraphql(source);
+  } catch {
+    return fallback;
+  }
+  const op = doc.definitions.find(
+    (d): d is OperationDefinitionNode => d.kind === Kind.OPERATION_DEFINITION,
+  );
+  if (!op) return fallback;
+  if (op.name?.value) return op.name.value;
+  // Anonymous — derive from first selection.
+  const firstField = op.selectionSet.selections.find(
+    (s) => s.kind === Kind.FIELD,
+  );
+  if (firstField && firstField.kind === Kind.FIELD) {
+    const opType = op.operation; // 'query' | 'mutation' | 'subscription'
+    const fieldName = firstField.name.value;
+    return `anon_${opType}_${fieldName}`;
+  }
+  return fallback;
+}
+
+/**
+ * Default per-test seed for the offline ShopState. Stable IDs so
+ * contracts captured from offline are deterministic across runs.
+ * Tests that need different data still seed via their own factories;
+ * contracts use this snapshot.
+ */
+function buildSeededState(): ShopState {
+  const state = new ShopState({
+    shop: {
+      domain: 'test-shop.myshopify.com',
+      permanent_domain: 'test-shop.myshopify.com',
+    },
+  });
+  state.addProduct({
+    id: 900_000_001,
+    handle: 'sample-product',
+    title: 'Sample Product',
+    description: 'Used by contract capture as a deterministic fixture.',
+    price: 1000,
+    vendor: 'Sample Vendor',
+    type: 'Sample',
+    variants: [
+      {
+        id: 900_010_001,
+        title: 'Default Title',
+        price: 1000,
+        available: true,
+        sku: 'SAMPLE-1',
+        inventory_quantity: 100,
+        selected_options: ['Default Title'],
+      },
+    ],
+    tags: [],
+  });
+  state.addCollection({
+    id: 900_020_001,
+    handle: 'sample-collection',
+    title: 'Sample Collection',
+    productHandles: ['sample-product'],
+  });
+  return state;
+}
+
+interface SynthesisedVariables {
+  values: Record<string, unknown>;
+  /** Variables we couldn't synthesise — operation will be marked partial. */
+  unhandled: string[];
+}
+
+/**
+ * Synthesise plausible variable values for an operation's variable
+ * definitions. Handles ID / String / Int / Float / Boolean scalars
+ * and their nullable + list variants. Complex input objects fall
+ * through to `unhandled` — the user can override via fixtures.json.
+ */
+function synthesiseVariables(
+  op: OperationDefinitionNode,
+  schema: GraphQLSchema,
+  state: ShopState,
+  override: Record<string, unknown>,
+): SynthesisedVariables {
+  const values: Record<string, unknown> = {};
+  const unhandled: string[] = [];
+  for (const def of op.variableDefinitions ?? []) {
+    const name = def.variable.name.value;
+    if (name in override) {
+      values[name] = override[name];
+      continue;
+    }
+    const synthValue = synthesiseTypeNode(def.type, schema, state);
+    if (synthValue === SYNTH_UNHANDLED) {
+      unhandled.push(name);
+    } else {
+      values[name] = synthValue;
+    }
+  }
+  return { values, unhandled };
+}
+
+const SYNTH_UNHANDLED = Symbol('unhandled');
+
+/**
+ * Convert an AST type-node (from a variable definition) to a
+ * resolved `GraphQLType`, then defer to the type-driven synthesiser.
+ *
+ * The two-step approach (AST → GraphQLType → value) is so that
+ * Input objects can be walked by their FIELD definitions, not by
+ * raw AST. Field types know exactly what's required vs optional, so
+ * we only fill required fields and leave optional ones absent.
+ */
+function synthesiseTypeNode(
+  typeNode: TypeNode,
+  schema: GraphQLSchema,
+  state: ShopState,
+): unknown | typeof SYNTH_UNHANDLED {
+  const resolved = typeFromAST(schema, typeNode);
+  if (!resolved) return SYNTH_UNHANDLED;
+  return synthesiseGraphQLType(resolved, state);
+}
+
+/**
+ * Recursively synthesise a sensible default for a GraphQLType.
+ *
+ *   - NonNull → required; recurse into inner type.
+ *   - List → empty array (valid for any list).
+ *   - Scalar → primitive defaults (ID = first seeded product GID,
+ *     others sensible per name).
+ *   - Enum → first value of the enum.
+ *   - InputObject → object with only its required fields filled;
+ *     optional fields left absent.
+ *
+ * Returns `SYNTH_UNHANDLED` only for types we genuinely can't
+ * produce a value for (custom scalars with non-obvious shapes etc.).
+ * Callers should treat that as "user must override via fixtures.json".
+ */
+function synthesiseGraphQLType(
+  type: GraphQLType,
+  state: ShopState,
+): unknown | typeof SYNTH_UNHANDLED {
+  if (isNonNullType(type)) {
+    return synthesiseGraphQLType(type.ofType, state);
+  }
+  if (isListType(type)) {
+    return [];
+  }
+  if (isScalarType(type)) {
+    const name = type.name;
+    if (name === 'ID') {
+      const first = Array.from(state.products.values())[0];
+      return first
+        ? `gid://shopify/Product/${first.id}`
+        : 'gid://shopify/Resource/1';
+    }
+    if (name === 'String') return 'sample';
+    if (name === 'Int') return 1;
+    if (name === 'Float') return 1.0;
+    if (name === 'Boolean') return true;
+    // Shopify-specific custom scalars we can stub with a primitive
+    // — these all serialise as strings on the wire.
+    if (
+      name === 'URL' ||
+      name === 'DateTime' ||
+      name === 'Date' ||
+      name === 'Decimal' ||
+      name === 'Money' ||
+      name === 'HTML' ||
+      name === 'JSON' ||
+      name === 'JSONString' ||
+      name === 'FormattedString' ||
+      name === 'StorefrontID' ||
+      name === 'UnsignedInt64'
+    ) {
+      if (name === 'URL') return 'https://example.com';
+      if (name === 'DateTime') return '2026-01-01T00:00:00Z';
+      if (name === 'Date') return '2026-01-01';
+      if (name === 'JSON' || name === 'JSONString') return '{}';
+      if (name === 'UnsignedInt64') return '1';
+      return '1.00'; // Decimal / Money / HTML / FormattedString — primitive default
+    }
+    return SYNTH_UNHANDLED;
+  }
+  if (isEnumType(type)) {
+    const values = type.getValues();
+    return values[0]?.value ?? SYNTH_UNHANDLED;
+  }
+  if (isInputObjectType(type)) {
+    const fields = type.getFields();
+    const out: Record<string, unknown> = {};
+    let anyRequired = false;
+    for (const [fieldName, field] of Object.entries(fields)) {
+      // Only fill REQUIRED fields. Optional ones get omitted —
+      // keeps the variable payload minimal + valid.
+      if (!isNonNullType(field.type)) continue;
+      anyRequired = true;
+      const value = synthesiseGraphQLType(field.type, state);
+      if (value === SYNTH_UNHANDLED) return SYNTH_UNHANDLED;
+      out[fieldName] = value;
+    }
+    // Edge case: input object with NO required fields. Empty object
+    // is a valid value.
+    return anyRequired ? out : {};
+  }
+  // Interfaces / Unions / custom scalars not covered above.
+  return SYNTH_UNHANDLED;
+}
+
+interface CapturedContract {
+  operationName: string;
+  source: string;
+  variables: Record<string, unknown>;
+  response: unknown;
+  capturedFrom: 'offline' | 'live';
+  capturedAt: string;
+  /** Diagnostic — set when capture didn't run cleanly. */
+  warning?: string;
+}
+
+/**
+ * Offline-only executor — POSTs the operation to an in-process
+ * Hono mock for the consuming app's API surface (admin or
+ * storefront GraphQL).
+ *
+ * No live target by design: contract capture at the consuming-app
+ * layer answers "what does the mock return for my operation?".
+ * The separate question — "does the mock match real Shopify on
+ * platform primitives?" — is owned by
+ * `@essential-apps/shopify-test-conformance` and runs against a
+ * canonical operation matrix, not per-app operation sets. An
+ * earlier version of this script had a `--target live` mode that
+ * mixed the two layers; removed for clarity.
+ */
+type Executor = (
+  source: string,
+  variables: Record<string, unknown>,
+) => Promise<unknown>;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Hono types are loose
+function buildOfflineExecutor(api: ApiType, state: ShopState): Executor {
+  const app =
+    api === 'admin' ? createAdminApi({ state }) : createStorefrontApi({ state });
+  const endpoint =
+    api === 'admin'
+      ? '/admin/api/2025-07/graphql.json'
+      : '/api/2025-07/graphql.json';
+  return async (source, variables) => {
+    const resp = await app.request(endpoint, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Shopify-Access-Token': 'mock-access-token',
+      },
+      body: JSON.stringify({ query: source, variables }),
+    });
+    return resp.json();
+  };
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs();
+
+  const sdl = args.api === 'admin' ? loadAdminSdl() : loadStorefrontSdl();
+  const schema = buildSchema(sdl);
+
+  // Load per-app fixtures if present (gives users a way to override
+  // synthesised variables for specific operations).
+  let fixtures: Record<string, Record<string, unknown>> = {};
+  if (existsSync(args.fixturesPath)) {
+    fixtures = JSON.parse(readFileSync(args.fixturesPath, 'utf8')) as Record<
+      string,
+      Record<string, unknown>
+    >;
+  }
+
+  const sourceDir = resolve(args.cwd, args.glob);
+  const files = Array.from(walkSources(sourceDir));
+  if (files.length === 0) {
+    console.error(`[capture-contracts] no source files under ${sourceDir}`);
+    process.exit(2);
+  }
+
+  // Collect operations + dedup by source (the same #graphql may
+  // appear in multiple files if shared via a util).
+  const opsByName = new Map<string, ExtractedOp>();
+  for (const f of files) {
+    const content = readFileSync(f, 'utf8');
+    for (const op of extractOperations(f, content)) {
+      const existing = opsByName.get(op.name);
+      if (!existing) opsByName.set(op.name, op);
+    }
+  }
+  if (!args.quiet) {
+    console.log(
+      `[capture-contracts] api=${args.api} ${files.length} file(s) scanned, ${opsByName.size} unique operation(s)`,
+    );
+  }
+
+  const state = buildSeededState();
+  const executor: Executor = buildOfflineExecutor(args.api, state);
+
+  mkdirSync(args.outDir, { recursive: true });
+
+  let captured = 0;
+  let skipped = 0;
+  let drift = 0;
+
+  for (const op of opsByName.values()) {
+    let doc: DocumentNode;
+    try {
+      doc = parseGraphql(op.source);
+    } catch (err) {
+      writeFileSync(
+        resolve(args.outDir, `${op.name}.json`),
+        JSON.stringify(
+          {
+            operationName: op.name,
+            source: op.source,
+            variables: {},
+            response: null,
+            capturedFrom: 'offline',
+            capturedAt: new Date().toISOString(),
+            warning: `parse failed: ${(err as Error).message}`,
+          } satisfies CapturedContract,
+          null,
+          2,
+        ) + '\n',
+      );
+      skipped++;
+      continue;
+    }
+
+    // Validate against schema before executing — operations that
+    // reference fields the SDL doesn't have are recorded as drift.
+    const validationErrors = validateGraphql(schema, doc);
+    if (validationErrors.length > 0) {
+      writeFileSync(
+        resolve(args.outDir, `${op.name}.json`),
+        JSON.stringify(
+          {
+            operationName: op.name,
+            source: op.source,
+            variables: {},
+            response: null,
+            capturedFrom: 'offline',
+            capturedAt: new Date().toISOString(),
+            warning: `schema validation failed: ${validationErrors
+              .map((e) => e.message)
+              .join(' | ')}`,
+          } satisfies CapturedContract,
+          null,
+          2,
+        ) + '\n',
+      );
+      drift++;
+      continue;
+    }
+
+    // Synthesise variables (with fixture overrides).
+    const opDef = doc.definitions.find(
+      (d): d is OperationDefinitionNode =>
+        d.kind === Kind.OPERATION_DEFINITION,
+    );
+    let variables: Record<string, unknown> = {};
+    let warning: string | undefined;
+    if (opDef) {
+      const synth = synthesiseVariables(
+        opDef,
+        schema,
+        state,
+        fixtures[op.name] ?? {},
+      );
+      variables = synth.values;
+      if (synth.unhandled.length > 0) {
+        warning =
+          `variables not synthesised: ${synth.unhandled.join(', ')}. ` +
+          `Add to ${relative(args.cwd, args.fixturesPath)} under "${op.name}" to capture this operation.`;
+      }
+    }
+
+    // If we couldn't fill required vars, record + skip execution.
+    if (warning) {
+      writeFileSync(
+        resolve(args.outDir, `${op.name}.json`),
+        JSON.stringify(
+          {
+            operationName: op.name,
+            source: op.source,
+            variables,
+            response: null,
+            capturedFrom: 'offline',
+            capturedAt: new Date().toISOString(),
+            warning,
+          } satisfies CapturedContract,
+          null,
+          2,
+        ) + '\n',
+      );
+      skipped++;
+      continue;
+    }
+
+    // Execute and capture.
+    let response: unknown;
+    try {
+      response = await executor(op.source, variables);
+    } catch (err) {
+      writeFileSync(
+        resolve(args.outDir, `${op.name}.json`),
+        JSON.stringify(
+          {
+            operationName: op.name,
+            source: op.source,
+            variables,
+            response: null,
+            capturedFrom: 'offline',
+            capturedAt: new Date().toISOString(),
+            warning: `execution failed: ${(err as Error).message}`,
+          } satisfies CapturedContract,
+          null,
+          2,
+        ) + '\n',
+      );
+      drift++;
+      continue;
+    }
+
+    writeFileSync(
+      resolve(args.outDir, `${op.name}.json`),
+      JSON.stringify(
+        {
+          operationName: op.name,
+          source: op.source,
+          variables,
+          response,
+          capturedFrom: 'offline',
+          capturedAt: new Date().toISOString(),
+        } satisfies CapturedContract,
+        null,
+        2,
+      ) + '\n',
+    );
+    captured++;
+  }
+
+  if (!args.quiet) {
+    console.log(
+      `[capture-contracts] ${captured} captured, ${skipped} skipped (need fixtures), ${drift} drift. Output: ${relative(args.cwd, args.outDir)}/`,
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(2);
+});
+
+// Silence the unused-typeinfo warning — the imports are used by the
+// validate / TypeInfo path the script extends in follow-ups.
+void TypeInfo;
+void visit;
+void visitWithTypeInfo;
+void dirname;
+void basename;
+type _UseT = GraphQLType;