@essential-apps/shopify-test-runner 1.0.0

package/src/scripts/captureRestContracts.ts

@@ -0,0 +1,319 @@
+#!/usr/bin/env node
+/**
+ * captureRestContracts — REST analog of `captureContracts`.
+ *
+ * Where the GraphQL capture extracts operations from `#graphql`
+ * template literals scattered through source, REST capture reads
+ * an explicit MANIFEST that the consuming app maintains at
+ * `tests/test-offline/contracts/rest.manifest.json`. Why manifest rather
+ * than static analysis:
+ *
+ *   1. Shopify apps overwhelmingly use the SDK resource-class
+ *      pattern: `admin.rest.resources.Theme.all({...})`. The
+ *      method-name → HTTP-path mapping lives inside
+ *      `@shopify/shopify-api`'s resource classes; reconstructing it
+ *      from app source requires reproducing that mapping table.
+ *      The manifest sidesteps the problem — devs declare the wire-
+ *      shape they actually want pinned.
+ *   2. REST endpoints aren't always tied to one call site (apps
+ *      may build paths dynamically). The manifest is the canonical
+ *      "this is the REST surface our app depends on" — same role
+ *      the GraphQL #graphql literals play.
+ *   3. The manifest is small (most apps have <20 REST endpoints).
+ *      Auto-extraction is a polish item; explicit manifest works
+ *      today.
+ *
+ * Manifest format:
+ *
+ *   {
+ *     "operations": [
+ *       {
+ *         "name": "listThemes",
+ *         "method": "GET",
+ *         "path": "/admin/api/{version}/themes.json",
+ *         "query": { "role": "main" }
+ *       },
+ *       {
+ *         "name": "getThemeAssets",
+ *         "method": "GET",
+ *         "path": "/admin/api/{version}/themes/{themeId}/assets.json",
+ *         "pathParams": { "themeId": "1" },
+ *         "query": { "asset[key]": "config/settings_data.json" }
+ *       }
+ *     ]
+ *   }
+ *
+ * `{version}` is substituted at run-time from the API version the
+ * mock serves; `{...}` other path params are substituted from
+ * `pathParams`. Bodies (for POST/PUT) live in `body`.
+ *
+ * Capture executes each entry against offline or live, vendors
+ * the response under `tests/test-offline/contracts/admin-rest/<name>.json`
+ * (alongside the GraphQL contracts) in the same shape
+ * `verifyContracts` consumes.
+ */
+import {
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  existsSync,
+  statSync,
+} from 'node:fs';
+import { resolve, relative } from 'node:path';
+import {
+  createAdminApi,
+} from '@essential-apps/shopify-test-shopify-api';
+import { ShopState } from '@essential-apps/shopify-test-storefront';
+
+interface Args {
+  manifestPath: string;
+  outDir: string;
+  cwd: string;
+  quiet: boolean;
+}
+
+interface RestOperation {
+  name: string;
+  method: 'GET' | 'POST' | 'PUT' | 'DELETE';
+  /** Path template. `{version}` substituted with apiVersion; other `{x}` from pathParams. */
+  path: string;
+  pathParams?: Record<string, string>;
+  query?: Record<string, string>;
+  body?: unknown;
+}
+
+interface RestManifest {
+  operations: RestOperation[];
+}
+
+const ADMIN_API_VERSION = '2025-07';
+
+function parseArgs(): Args {
+  const argv = process.argv.slice(2);
+  const out: Args = {
+    manifestPath: '',
+    outDir: '',
+    cwd: process.cwd(),
+    quiet: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i] ?? '';
+    if (a === '--manifest' && i + 1 < argv.length) {
+      out.manifestPath = argv[++i] ?? '';
+    } else if (a === '--out' && i + 1 < argv.length) {
+      out.outDir = argv[++i] ?? '';
+    } else if (a === '--quiet') {
+      out.quiet = true;
+    } else {
+      console.error(`unknown arg: ${a}`);
+      process.exit(2);
+    }
+  }
+  if (!out.manifestPath) {
+    out.manifestPath = resolve(out.cwd, 'tests/test-offline/contracts/rest.manifest.json');
+  } else {
+    out.manifestPath = resolve(out.cwd, out.manifestPath);
+  }
+  if (!out.outDir) {
+    out.outDir = resolve(out.cwd, 'tests/test-offline/contracts/admin-rest');
+  } else {
+    out.outDir = resolve(out.cwd, out.outDir);
+  }
+  return out;
+}
+
+interface RestExecutor {
+  (op: RestOperation): Promise<{ status: number; headers: Record<string, string>; body: unknown }>;
+}
+
+/** Substitute `{x}` placeholders in a path. */
+function fillPath(template: string, params: Record<string, string>): string {
+  return template.replace(/\{([^}]+)\}/g, (_, key) => {
+    if (!(key in params)) {
+      throw new Error(`path template references {${key}} but no value in pathParams`);
+    }
+    return encodeURIComponent(params[key] ?? '');
+  });
+}
+
+function appendQuery(path: string, query?: Record<string, string>): string {
+  if (!query || Object.keys(query).length === 0) return path;
+  const params = new URLSearchParams();
+  for (const [k, v] of Object.entries(query)) params.set(k, v);
+  const joiner = path.includes('?') ? '&' : '?';
+  return `${path}${joiner}${params.toString()}`;
+}
+
+/**
+ * In-process Hono executor — hits the mock-admin REST handlers
+ * directly via `app.request(...)`. No port binding, no auth gate
+ * (the mock accepts any X-Shopify-Access-Token).
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Hono is loosely typed
+function buildOfflineExecutor(state: ShopState): RestExecutor {
+  const app = createAdminApi({ state });
+  return async (op) => {
+    const filled = fillPath(op.path, {
+      version: ADMIN_API_VERSION,
+      ...(op.pathParams ?? {}),
+    });
+    const finalPath = appendQuery(filled, op.query);
+    const init: RequestInit = {
+      method: op.method,
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Shopify-Access-Token': 'mock-access-token',
+      },
+    };
+    if (op.body !== undefined) {
+      init.body = JSON.stringify(op.body);
+    }
+    const resp: Response = await (app.request as any)(finalPath, init);
+    const headers: Record<string, string> = {};
+    resp.headers.forEach((v, k) => { headers[k] = v; });
+    let body: unknown = null;
+    try {
+      body = await resp.json();
+    } catch {
+      // Some endpoints return empty/non-JSON. Capture as null.
+    }
+    return { status: resp.status, headers, body };
+  };
+}
+
+// Live executor intentionally removed — platform parity (does the
+// offline mock match real Shopify) is owned by
+// `@essential-apps/shopify-test-conformance`. Consuming-app contract
+// capture only targets the offline mock.
+
+interface CapturedRestContract {
+  operationName: string;
+  protocol: 'rest';
+  method: string;
+  path: string;
+  pathParams?: Record<string, string>;
+  query?: Record<string, string>;
+  body?: unknown;
+  response: {
+    status: number;
+    body: unknown;
+  };
+  capturedFrom: 'offline';
+  capturedAt: string;
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs();
+  if (!existsSync(args.manifestPath)) {
+    console.error(
+      `[capture-rest] manifest not found at ${relative(args.cwd, args.manifestPath)}\n` +
+        `Create one — see docs/CONTRACTS.md for the schema.`,
+    );
+    process.exit(2);
+  }
+  let st;
+  try {
+    st = statSync(args.manifestPath);
+  } catch {
+    console.error(`[capture-rest] cannot stat ${args.manifestPath}`);
+    process.exit(2);
+  }
+  if (!st.isFile()) {
+    console.error(`[capture-rest] ${args.manifestPath} is not a file`);
+    process.exit(2);
+  }
+  const manifest = JSON.parse(readFileSync(args.manifestPath, 'utf8')) as RestManifest;
+  if (!manifest.operations || !Array.isArray(manifest.operations)) {
+    console.error(`[capture-rest] manifest must have an "operations" array`);
+    process.exit(2);
+  }
+
+  const state = buildSeededState();
+  const executor: RestExecutor =
+    buildOfflineExecutor(state);
+
+  mkdirSync(args.outDir, { recursive: true });
+
+  if (!args.quiet) {
+    console.log(
+      `[capture-rest] ${manifest.operations.length} operation(s) in manifest`,
+    );
+  }
+
+  let captured = 0;
+  let failed = 0;
+
+  for (const op of manifest.operations) {
+    let response;
+    try {
+      response = await executor(op);
+    } catch (err) {
+      failed++;
+      writeFileSync(
+        resolve(args.outDir, `${op.name}.json`),
+        JSON.stringify(
+          {
+            operationName: op.name,
+            protocol: 'rest' as const,
+            method: op.method,
+            path: op.path,
+            ...(op.pathParams ? { pathParams: op.pathParams } : {}),
+            ...(op.query ? { query: op.query } : {}),
+            ...(op.body !== undefined ? { body: op.body } : {}),
+            response: { status: 0, body: null },
+            capturedFrom: 'offline',
+            capturedAt: new Date().toISOString(),
+            warning: `executor failed: ${(err as Error).message}`,
+          },
+          null,
+          2,
+        ) + '\n',
+      );
+      continue;
+    }
+    const contract: CapturedRestContract = {
+      operationName: op.name,
+      protocol: 'rest',
+      method: op.method,
+      path: op.path,
+      ...(op.pathParams ? { pathParams: op.pathParams } : {}),
+      ...(op.query ? { query: op.query } : {}),
+      ...(op.body !== undefined ? { body: op.body } : {}),
+      response: { status: response.status, body: response.body },
+      capturedFrom: 'offline',
+      capturedAt: new Date().toISOString(),
+    };
+    writeFileSync(
+      resolve(args.outDir, `${op.name}.json`),
+      JSON.stringify(contract, null, 2) + '\n',
+    );
+    captured++;
+  }
+
+  if (!args.quiet) {
+    console.log(
+      `[capture-rest] ${captured} captured, ${failed} failed. Output: ${relative(args.cwd, args.outDir)}/`,
+    );
+  }
+}
+
+/**
+ * Deterministic ShopState seed — same baseline as
+ * captureContracts.ts (the GraphQL counterpart). The REST handlers
+ * in mock-admin already serve sensible defaults regardless of state
+ * (themes endpoint returns a stub theme even with empty state); we
+ * still construct a state in case future endpoints depend on it.
+ */
+function buildSeededState(): ShopState {
+  return new ShopState({
+    shop: {
+      domain: 'test-shop.myshopify.com',
+      permanent_domain: 'test-shop.myshopify.com',
+    },
+  });
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(2);
+});

package/src/scripts/checkOperationCoverage.ts

@@ -0,0 +1,365 @@
+#!/usr/bin/env node
+/**
+ * checkOperationCoverage — drift check between a consuming app's
+ * GraphQL operations and the offline mock's resolver coverage.
+ *
+ * The problem this catches: production code calls
+ *   `admin.graphql(\`#graphql query { product(id: $id) { title } }\`)`
+ * but the offline mock has no `Query.product` resolver. Real Shopify
+ * answers fine — offline returns `null` silently, and the test
+ * fails several layers downstream (form sits in a skeleton-loading
+ * state, etc.) with no hint that the root cause is a missing mock.
+ *
+ * The fix this enforces: every Query/Mutation root field referenced
+ * by production code MUST have a resolver in the corresponding
+ * offline mock's resolver registry. Run this in CI before tests
+ * and the gap surfaces at design time, not at test failure time.
+ *
+ * Single source of truth: the SDL files in
+ * `@essential-apps/shopify-test-shopify-api` (which the conformance
+ * suite proves match live Shopify) plus the consuming app's own
+ * source code. The mock resolver registry is what gets validated —
+ * if conformance says Shopify has Field X and your app calls X but
+ * we have no X resolver, this surfaces it.
+ *
+ * Usage (from a consuming app's repo root):
+ *
+ *   npx shopify-test-check-operation-coverage
+ *   npx shopify-test-check-operation-coverage --api admin
+ *   npx shopify-test-check-operation-coverage --api storefront
+ *   npx shopify-test-check-operation-coverage --glob "./app/**\/*.ts"
+ *
+ * Exit codes:
+ *   0 — all referenced root fields have resolvers
+ *   1 — gaps found (printed with file:line citations)
+ *   2 — bad invocation / parse error
+ *
+ * Scope of the check: Query/Mutation root fields only. Nested object
+ * fields are not checked — graphql-tools defaults to property-access
+ * for fields without an explicit resolver, so they're "covered" as
+ * long as the parent resolver returned a shape with that key. The
+ * silent-null failure mode only affects ROOT resolvers (Query.X
+ * with no resolver returns null even though the field is nullable
+ * — exactly the bug this catches).
+ *
+ * Why a CLI vs a Vitest test: this needs to read the consuming
+ * app's source (path-dependent) AND the mock's resolvers (package
+ * import). A CLI is the natural shape — every consuming app's CI
+ * runs it the same way.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { extname, relative, resolve } from 'node:path';
+import {
+  buildSchema,
+  parse,
+  TypeInfo,
+  visit,
+  visitWithTypeInfo,
+  type GraphQLSchema,
+} from 'graphql';
+import {
+  adminResolvers,
+  storefrontResolvers,
+  loadAdminSdl,
+  loadStorefrontSdl,
+} from '@essential-apps/shopify-test-shopify-api';
+
+type ApiType = 'admin' | 'storefront';
+
+interface Args {
+  api: ApiType;
+  glob: string;
+  cwd: string;
+  json: boolean;
+}
+
+function parseArgs(): Args {
+  const argv = process.argv.slice(2);
+  const out: Args = {
+    api: 'admin',
+    glob: process.env['SHOPIFY_TEST_OPS_GLOB'] ?? './app',
+    cwd: process.cwd(),
+    json: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i] ?? '';
+    if (a === '--api' && i + 1 < argv.length) {
+      const next = argv[++i] ?? '';
+      if (next !== 'admin' && next !== 'storefront') {
+        console.error(`--api must be "admin" or "storefront" (got "${next}")`);
+        process.exit(2);
+      }
+      out.api = next;
+    } else if (a === '--glob' && i + 1 < argv.length) {
+      out.glob = argv[++i] ?? out.glob;
+    } else if (a === '--json') {
+      out.json = true;
+    } else if (a === '--help' || a === '-h') {
+      printUsage();
+      process.exit(0);
+    } else {
+      console.error(`unknown arg: ${a}`);
+      printUsage();
+      process.exit(2);
+    }
+  }
+  return out;
+}
+
+function printUsage(): void {
+  console.error(
+    [
+      'usage: shopify-test-check-operation-coverage [--api admin|storefront] [--glob <dir>] [--json]',
+      '',
+      'Walks every .ts/.tsx/.js/.jsx file under --glob, extracts inline',
+      '#graphql template literals, and verifies every Query/Mutation root',
+      'field referenced has a resolver in the offline mock registry.',
+      '',
+      '--api      which mock to check (default: admin)',
+      '--glob     directory to scan (default: ./app)',
+      '--json     emit a structured JSON report on stdout',
+    ].join('\n'),
+  );
+}
+
+/**
+ * Walk a directory recursively, returning every file whose extension
+ * matches `.ts | .tsx | .js | .jsx`. We don't depend on `glob` — the
+ * extra dep would be overkill for "walk one tree and filter".
+ */
+function* walkSources(root: string): Generator<string> {
+  let entries: string[];
+  try {
+    entries = readdirSync(root);
+  } catch {
+    return;
+  }
+  for (const name of entries) {
+    if (name === 'node_modules' || name === 'dist' || name === '.git') continue;
+    const full = resolve(root, name);
+    let st;
+    try {
+      st = statSync(full);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) {
+      yield* walkSources(full);
+    } else if (st.isFile()) {
+      const ext = extname(name);
+      if (ext === '.ts' || ext === '.tsx' || ext === '.js' || ext === '.jsx') {
+        yield full;
+      }
+    }
+  }
+}
+
+interface ExtractedOp {
+  file: string;
+  line: number;
+  source: string;
+}
+
+/**
+ * Extract every `#graphql ...` tagged template literal from a source
+ * file. We accept three common conventions:
+ *
+ *   admin.graphql(`#graphql ...`)            // shopify-app-remix
+ *   gql`#graphql ...`                        // codegen tag
+ *   `#graphql ...`                           // raw template
+ *
+ * Real-world false negatives we accept: operations not tagged
+ * `#graphql`, operations split across template-string interpolation,
+ * dynamic strings built at runtime. Those bypass codegen too, so
+ * the consuming app is already non-conformant — the warning is OK.
+ */
+function extractOperations(file: string, content: string): ExtractedOp[] {
+  const ops: ExtractedOp[] = [];
+  // Match a backtick-delimited string whose FIRST non-whitespace token
+  // is `#graphql`. The `[^\`]*` keeps it simple — operations with
+  // backticks inside string values aren't valid GraphQL, so this is
+  // safe in practice.
+  const regex = /`\s*#graphql\b([^`]*)`/g;
+  let m: RegExpExecArray | null;
+  while ((m = regex.exec(content)) !== null) {
+    const line = content.slice(0, m.index).split('\n').length;
+    ops.push({ file, line, source: m[1] ?? '' });
+  }
+  return ops;
+}
+
+interface FieldRef {
+  type: string;
+  field: string;
+  /** Where this field was first referenced (file:line). */
+  origin: { file: string; line: number };
+}
+
+/**
+ * Walk a GraphQL document and return every (Type, Field) pair the
+ * selection set touches. We use `TypeInfo` to track the parent type
+ * at each `Field` visit, which is the only way to know the
+ * abstract-type a field belongs to without re-implementing the
+ * resolver lookup logic.
+ */
+function collectFieldRefs(
+  source: string,
+  schema: GraphQLSchema,
+  file: string,
+  line: number,
+  out: Map<string, FieldRef>,
+  parseErrors: { file: string; line: number; message: string }[],
+): void {
+  let doc;
+  try {
+    doc = parse(source);
+  } catch (err) {
+    parseErrors.push({ file, line, message: (err as Error).message });
+    return;
+  }
+  const typeInfo = new TypeInfo(schema);
+  visit(
+    doc,
+    visitWithTypeInfo(typeInfo, {
+      Field() {
+        const parentType = typeInfo.getParentType();
+        const fieldDef = typeInfo.getFieldDef();
+        if (!parentType || !fieldDef) return;
+        const key = `${parentType.name}.${fieldDef.name}`;
+        if (!out.has(key)) {
+          out.set(key, {
+            type: parentType.name,
+            field: fieldDef.name,
+            origin: { file, line },
+          });
+        }
+      },
+    }),
+  );
+}
+
+/**
+ * Resolver registries are `{ [TypeName]: { [FieldName]: ResolverFn } }`.
+ * Build a flat set of "TypeName.FieldName" strings for fast lookup.
+ */
+function flattenResolvers(
+  resolvers: Record<string, Record<string, unknown>>,
+): Set<string> {
+  const out = new Set<string>();
+  for (const [typeName, fields] of Object.entries(resolvers)) {
+    if (!fields || typeof fields !== 'object') continue;
+    for (const fieldName of Object.keys(fields)) {
+      out.add(`${typeName}.${fieldName}`);
+    }
+  }
+  return out;
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs();
+
+  const sdl = args.api === 'admin' ? loadAdminSdl() : loadStorefrontSdl();
+  const schema = buildSchema(sdl);
+
+  const sourceDir = resolve(args.cwd, args.glob);
+  const files = Array.from(walkSources(sourceDir));
+  if (files.length === 0) {
+    console.error(`[coverage] no source files under ${sourceDir}`);
+    process.exit(2);
+  }
+
+  const refs = new Map<string, FieldRef>();
+  const parseErrors: { file: string; line: number; message: string }[] = [];
+  let opCount = 0;
+  for (const f of files) {
+    const content = readFileSync(f, 'utf8');
+    for (const op of extractOperations(f, content)) {
+      opCount += 1;
+      collectFieldRefs(op.source, schema, op.file, op.line, refs, parseErrors);
+    }
+  }
+
+  const resolvers =
+    args.api === 'admin'
+      ? (adminResolvers as Record<string, Record<string, unknown>>)
+      : (storefrontResolvers as Record<string, Record<string, unknown>>);
+  const implemented = flattenResolvers(resolvers);
+
+  // Root types in the Shopify schemas: Admin's Query type is named
+  // `QueryRoot` and Mutation is `Mutation`; Storefront uses `QueryRoot`
+  // / `MutationRoot`. We get the actual names from the schema.
+  const queryRoot = schema.getQueryType()?.name;
+  const mutationRoot = schema.getMutationType()?.name;
+  const rootTypes = new Set<string>([
+    ...(queryRoot ? [queryRoot] : []),
+    ...(mutationRoot ? [mutationRoot] : []),
+  ]);
+
+  const uncoveredRoot: FieldRef[] = [];
+  for (const ref of refs.values()) {
+    if (!rootTypes.has(ref.type)) continue; // ignore nested fields
+    if (!implemented.has(`${ref.type}.${ref.field}`)) {
+      uncoveredRoot.push(ref);
+    }
+  }
+  uncoveredRoot.sort((a, b) => `${a.type}.${a.field}`.localeCompare(`${b.type}.${b.field}`));
+
+  if (args.json) {
+    process.stdout.write(
+      JSON.stringify(
+        {
+          api: args.api,
+          filesScanned: files.length,
+          opsFound: opCount,
+          rootFieldsReferenced: [...refs.values()].filter((r) =>
+            rootTypes.has(r.type),
+          ).length,
+          rootFieldsImplemented: [...implemented].filter((k) => {
+            const root = k.split('.')[0];
+            return root !== undefined && rootTypes.has(root);
+          }).length,
+          uncoveredRoot: uncoveredRoot.map((u) => ({
+            type: u.type,
+            field: u.field,
+            origin: u.origin,
+          })),
+          parseErrors,
+        },
+        null,
+        2,
+      ),
+    );
+    process.stdout.write('\n');
+  } else {
+    const cwd = args.cwd;
+    console.log(
+      `[coverage] api=${args.api}, scanned ${files.length} file(s), found ${opCount} operation(s)`,
+    );
+    if (parseErrors.length > 0) {
+      console.log(`[coverage] ${parseErrors.length} parse error(s):`);
+      for (const e of parseErrors) {
+        console.log(`  ! ${relative(cwd, e.file)}:${e.line} — ${e.message}`);
+      }
+    }
+    if (uncoveredRoot.length === 0) {
+      console.log('[coverage] ✓ every referenced root field has a resolver');
+    } else {
+      console.log(`[coverage] ✗ ${uncoveredRoot.length} uncovered root field(s):`);
+      for (const u of uncoveredRoot) {
+        console.log(
+          `  ${u.type}.${u.field}   (${relative(cwd, u.origin.file)}:${u.origin.line})`,
+        );
+      }
+      console.log('');
+      console.log(
+        `Fix: add a resolver in @essential-apps/shopify-test-shopify-api/src/${args.api}/resolvers/<file>.ts`,
+      );
+    }
+  }
+  process.exit(uncoveredRoot.length === 0 ? 0 : 1);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(2);
+});