@essential-apps/shopify-test-runner 1.0.12 → 1.0.14

package/src/scripts/captureSharedContracts.ts

@@ -0,0 +1,228 @@
+#!/usr/bin/env node
+/**
+ * Regenerate the SHARED contract goldens in
+ * `@essential-apps/shopify-test-contracts/fixtures` from the in-process
+ * offline mock — the "replayable-variable capture" generator.
+ *
+ * Why this exists (the design the centralization landed on):
+ *   A verify golden must be *mock-replayable* — `verify-contracts`
+ *   replays its `source` + `variables` against the offline mock and
+ *   asserts the response matches. A conformance capture (scrubbed LIVE
+ *   variables like `ownerId: <NUMERIC_ID>`, a richer field selection)
+ *   is NOT replayable: the mock can't resolve the placeholders. So the
+ *   two are different artifacts.
+ *
+ *   The clean split:
+ *     - The verify golden is captured FROM THE MOCK with deterministic
+ *       seed variables — replayable by construction. That's this script.
+ *     - `@essential-apps/shopify-test-conformance` separately CERTIFIES
+ *       the mock matches live Shopify (schema SDL + shape diffs). A
+ *       golden that's mock-replayable AND mock-certified-vs-live is
+ *       trustworthy end to end.
+ *
+ * This script keeps each golden's `operationName` / `source` /
+ * `variables` (so the *request* contract is stable) and refreshes only
+ * the `response` to the current mock output. Idempotent: unchanged
+ * responses aren't rewritten.
+ *
+ *   node --import tsx packages/runner/src/scripts/captureSharedContracts.ts
+ *   # or the bin: shopify-test-capture-shared-contracts
+ */
+import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import {
+  createAdminApi,
+  createStorefrontApi,
+} from '@essential-apps/shopify-test-shopify-api';
+import { ShopState } from '@essential-apps/shopify-test-storefront';
+import {
+  fixturesDir,
+  type OperationContract,
+  type RestContract,
+} from '@essential-apps/shopify-test-contracts/operation-contract';
+import { normaliseResponse } from '../contracts/normalize.js';
+
+const ADMIN_API_VERSION = '2025-07';
+
+/**
+ * Structural equality under the same normalisation `verify` applies
+ * (gids → `<ID>`, timestamps → `<DATETIME>`, …). We rewrite a golden
+ * only on genuine structural drift — not when a volatile-but-equivalent
+ * value (a fresh timestamp/id) changed — so the generator is idempotent.
+ */
+function sameShape(a: unknown, b: unknown): boolean {
+  return (
+    JSON.stringify(normaliseResponse(a)) === JSON.stringify(normaliseResponse(b))
+  );
+}
+
+/**
+ * Deterministic GraphQL seed. MUST stay in lockstep with
+ * `verifyContracts.ts` / `captureContracts.ts` `buildSeededState` — the
+ * goldens are replayed against this exact state at verify time.
+ */
+function buildGraphqlState(): ShopState {
+  const state = new ShopState({
+    shop: {
+      domain: 'test-shop.myshopify.com',
+      permanent_domain: 'test-shop.myshopify.com',
+    },
+  });
+  state.addProduct({
+    id: 900_000_001,
+    handle: 'sample-product',
+    title: 'Sample Product',
+    description: 'Used by contract capture as a deterministic fixture.',
+    price: 1000,
+    vendor: 'Sample Vendor',
+    type: 'Sample',
+    variants: [
+      {
+        id: 900_010_001,
+        title: 'Default Title',
+        price: 1000,
+        available: true,
+        sku: 'SAMPLE-1',
+        inventory_quantity: 100,
+        selected_options: ['Default Title'],
+      },
+    ],
+    tags: [],
+  });
+  state.addCollection({
+    id: 900_020_001,
+    handle: 'sample-collection',
+    title: 'Sample Collection',
+    productHandles: ['sample-product'],
+  });
+  return state;
+}
+
+/** REST seed — matches `verifyRestContracts.ts` `buildSeededState`. */
+function buildRestState(): ShopState {
+  return new ShopState({
+    shop: {
+      domain: 'test-shop.myshopify.com',
+      permanent_domain: 'test-shop.myshopify.com',
+    },
+  });
+}
+
+function fillPath(template: string, params: Record<string, string>): string {
+  return template.replace(/\{([^}]+)\}/g, (_, key: string) => {
+    if (!(key in params)) {
+      throw new Error(`path template references {${key}} but no value provided`);
+    }
+    return encodeURIComponent(params[key] ?? '');
+  });
+}
+function appendQuery(path: string, query?: Record<string, string>): string {
+  if (!query || Object.keys(query).length === 0) return path;
+  const params = new URLSearchParams();
+  for (const [k, v] of Object.entries(query)) params.set(k, v);
+  return `${path}${path.includes('?') ? '&' : '?'}${params.toString()}`;
+}
+
+/** Flat `<op>.json` goldens only (skip manifests + store-keyed captures). */
+function flatGoldens(dir: string): string[] {
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  return entries.filter(
+    (f) =>
+      f.endsWith('.json') &&
+      !f.endsWith('.manifest.json') &&
+      !f.slice(0, -'.json'.length).includes('.'),
+  );
+}
+
+async function regenGraphql(api: 'admin' | 'storefront'): Promise<number> {
+  const dir = resolve(fixturesDir(), api);
+  const state = buildGraphqlState();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Hono types are loose
+  const app: any =
+    api === 'admin' ? createAdminApi({ state }) : createStorefrontApi({ state });
+  const endpoint =
+    api === 'admin'
+      ? '/admin/api/2025-07/graphql.json'
+      : '/api/2025-07/graphql.json';
+  let changed = 0;
+  for (const f of flatGoldens(dir)) {
+    const p = resolve(dir, f);
+    const g = JSON.parse(readFileSync(p, 'utf8')) as OperationContract;
+    if (g.warning) continue;
+    const resp = await app.request(endpoint, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Shopify-Access-Token': 'mock-access-token',
+      },
+      body: JSON.stringify({ query: g.source, variables: g.variables }),
+    });
+    const actual = await resp.json();
+    if (!sameShape(g.response, actual)) {
+      g.response = actual;
+      writeFileSync(p, JSON.stringify(g, null, 2) + '\n');
+      console.log(`  updated ${api}/${f}`);
+      changed++;
+    }
+  }
+  return changed;
+}
+
+async function regenRest(): Promise<number> {
+  const dir = resolve(fixturesDir(), 'admin-rest');
+  const state = buildRestState();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Hono types are loose
+  const app: any = createAdminApi({ state });
+  let changed = 0;
+  for (const f of flatGoldens(dir)) {
+    const p = resolve(dir, f);
+    const c = JSON.parse(readFileSync(p, 'utf8')) as RestContract;
+    if (c.warning) continue;
+    const finalPath = appendQuery(
+      fillPath(c.path, { version: ADMIN_API_VERSION, ...(c.pathParams ?? {}) }),
+      c.query,
+    );
+    const init: RequestInit = {
+      method: c.method,
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Shopify-Access-Token': 'mock-access-token',
+      },
+    };
+    if (c.body !== undefined) init.body = JSON.stringify(c.body);
+    const resp: Response = await app.request(finalPath, init);
+    let body: unknown = null;
+    try {
+      body = await resp.json();
+    } catch {
+      /* empty / non-JSON */
+    }
+    const next = { status: resp.status, body };
+    if (c.response.status !== next.status || !sameShape(c.response.body, next.body)) {
+      c.response = next;
+      writeFileSync(p, JSON.stringify(c, null, 2) + '\n');
+      console.log(`  updated admin-rest/${f}`);
+      changed++;
+    }
+  }
+  return changed;
+}
+
+async function main(): Promise<void> {
+  console.log('[capture-shared-contracts] regenerating shared goldens from the offline mock…');
+  let total = 0;
+  total += await regenGraphql('admin');
+  total += await regenGraphql('storefront');
+  total += await regenRest();
+  console.log(`[capture-shared-contracts] done — ${total} golden(s) refreshed.`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/scripts/runOffline.ts

@@ -56,6 +56,7 @@ import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { pickApp, prepareOciArchive, envFileArgs, computeBuildHash } from '@essential-apps/shopify-test-core';
 import { buildExtensions } from '../lib/buildExtensions.js';
+import { exposeGuestVnc } from '../lib/guestVnc.js';
 
 const repoRoot = process.cwd();
 
@@ -338,6 +339,27 @@ async function main(): Promise<void> {
     }
   })();
   const runtimeTag = `sm${runtimeVersion.replace(/[^0-9a-z]+/gi, '')}`;
+
+  // Fold THIS runner package's own version into the snapshot key. The
+  // warm snapshot bakes the in-VM `npm install` (which installs the
+  // orchestrator + fixtures this same shopify-test version ships). Those
+  // run INSIDE the VM, so a version bump that changes orchestrator/
+  // fixture behaviour (e.g. screen-video support) would otherwise
+  // warm-restore a STALE in-VM install — the consumer updates the
+  // package but keeps running the old in-VM code until they manually
+  // `rm -rf ~/.cache/<app>-test`. Keying on the runner version makes a
+  // bump auto-rebake (cold install of the new version) — same
+  // no-manual-bust philosophy as manifestHash + runtimeTag.
+  const runnerVersion = ((): string => {
+    try {
+      return (
+        createRequire(import.meta.url)('../../package.json') as { version: string }
+      ).version;
+    } catch {
+      return 'unknown';
+    }
+  })();
+  const runnerTag = `r${runnerVersion.replace(/[^0-9a-z]+/gi, '')}`;
   // Scope the snapshot to the consuming app. supermachine keys WARM
   // snapshots by (image + warmupTag), but the image is shared across
   // apps and the rest of the tag is identical for any two apps vendoring
@@ -347,7 +369,7 @@ async function main(): Promise<void> {
   // identity (observed: essential-upsell booting with essential-seo's
   // DB). The cache paths are already appName-namespaced; this aligns the
   // snapshot key with them.
-  const warmupTag = `offline-v13-${appName}-${runtimeTag}-${manifestHash}`;
+  const warmupTag = `offline-v13-${appName}-${runtimeTag}-${runnerTag}-${manifestHash}`;
 
   // Dynamic import — @supermachine/core is heavy and only needed
   // when actually running tests, not for `--help` etc.
@@ -513,6 +535,15 @@ async function main(): Promise<void> {
       // orchestrator's Xvfb startup + the offlineFullStack screen-video
       // fixture).
       'TEST_OFFLINE_VIDEO',
+      // Offline runs headed-under-Xvfb by default; opt out to headless.
+      'TEST_OFFLINE_HEADLESS',
+      // Interactive explore mode: orchestrator skips Playwright and
+      // instead opens a headed Chromium (admin + storefront tabs) on
+      // the guest Xvfb display, which we surface over VNC (see the
+      // explore branch below).
+      'TEST_OFFLINE_EXPLORE',
+      'TEST_OFFLINE_EXPLORE_URL',
+      'TEST_OFFLINE_EXPLORE_NO_DEVTOOLS',
       'CI',
       'DEBUG',
     ]) {
@@ -585,6 +616,56 @@ async function main(): Promise<void> {
     forwardEnv['PRISMA_QUERY_ENGINE_LIBRARY'] =
       '/workspace/node_modules/.prisma/client/libquery_engine-linux-arm64-openssl-3.0.x.so.node';
 
+    // ── Explore mode (interactive, VNC) ───────────────────────────
+    // `TEST_OFFLINE_EXPLORE=true` boots the full offline mock stack
+    // and hands the developer a real browser inside the VM instead of
+    // running Playwright. We surface the guest's Xvfb display over VNC
+    // (supermachine `vm.exposeTcp`, NOT docker --publish) so they can
+    // click through the admin app + storefront against the mock stack.
+    // The orchestrator opens both tabs (see runOfflineFullTests explore
+    // branch) and holds until the browser window is closed / Ctrl-C.
+    if (process.env['TEST_OFFLINE_EXPLORE'] === 'true') {
+      let forwarder: { stop?: () => Promise<void> | void } | undefined;
+      try {
+        forwarder = (await exposeGuestVnc(vm, {
+          logPrefix: '[runOffline]',
+        })) as { stop?: () => Promise<void> | void };
+
+        console.error('[runOffline] launching explore session…');
+        const exploreProc = await vm.spawn({
+          argv: ['sh', '-c', `
+            cd /workspace
+            export PATH=/workspace/node_modules/.bin:$PATH
+            export DISPLAY=:99
+            node ${envFileArgs(repoRoot)} node_modules/@essential-apps/shopify-test-runner/dist/scripts/runOfflineFullTests.js
+          `],
+          env: forwardEnv,
+        });
+
+        // Stream guest output verbatim with NO stall/hard timeout — an
+        // explore session legitimately sits idle for as long as the
+        // developer is poking around. It ends when they close the
+        // browser (orchestrator exits) or Ctrl-C here.
+        const pump = (async () => {
+          while (true) {
+            const out = await exploreProc.readStdout(64 * 1024);
+            if (out.length === 0) {
+              await new Promise((r) => setTimeout(r, 200));
+              continue;
+            }
+            process.stdout.write(out);
+          }
+        })();
+        pump.catch(() => {});
+
+        const wait = await exploreProc.wait();
+        process.exitCode = wait.exitCode ?? 0;
+        return;
+      } finally {
+        if (forwarder?.stop) await Promise.resolve(forwarder.stop()).catch(() => {});
+      }
+    }
+
     // Optional --grep forwarding for fast iteration on a single
     // scenario or spec without writing a one-off bench script.
     const grep = process.env['PLAYWRIGHT_GREP'];

package/src/scripts/runOfflineFullTests.ts

@@ -1229,13 +1229,15 @@ async function main(): Promise<void> {
 
     // ── Explore mode (no Playwright; manual click-around) ──
     // When `TEST_OFFLINE_EXPLORE=true` is set, skip Playwright
-    // entirely. Instead, launch a single non-headless Chrome on the
-    // container's Xvfb display, pointed at the admin app's entry
-    // URL. The container is started with `--publish 5900:5900` +
-    // `TEST_ONLINE_VNC=1`, so the developer can connect from macOS via
+    // entirely. Instead, launch a non-headless Chrome on the guest's
+    // Xvfb :99 display with TWO tabs — the admin app and the storefront
+    // — both pointed at the in-VM mock stack. The host-side runner
+    // (`runOffline.ts`) surfaces :99 over VNC via supermachine's
+    // `vm.exposeTcp` (NOT docker --publish — we don't use docker for
+    // anything), so the developer connects from macOS via
     //   open vnc://localhost:5900   (password: `test`)
-    // and click through the admin and storefront against the online
-    // mock stack. The process holds open until SIGINT/SIGTERM.
+    // and clicks through admin + storefront against the mock stack.
+    // The process holds open until the browser is closed / SIGINT.
     //
     // This is the "interactive Shopify dev store" mode — useful
     // for visually inspecting funnel rendering, debugging admin
@@ -1342,6 +1344,26 @@ async function main(): Promise<void> {
         exploreCtx.pages()[0] ?? (await exploreCtx.newPage());
       await explorePage.goto(adminUrl, { waitUntil: 'domcontentloaded' });
 
+      // Second tab: the storefront (app embed + widget render against
+      // the seeded catalog). Opening both up-front means the developer
+      // lands on a ready admin + storefront without typing URLs. Errors
+      // are swallowed — a slow/failed storefront load shouldn't abort
+      // the whole explore session (the admin tab is still useful).
+      const storefrontUrl =
+        process.env['TEST_OFFLINE_EXPLORE_STOREFRONT_URL'] ??
+        `https://${DEFAULT_SHOP_DOMAIN}/`;
+      try {
+        const sfPage = await exploreCtx.newPage();
+        await sfPage.goto(storefrontUrl, { waitUntil: 'domcontentloaded' });
+      } catch (err) {
+        console.error(
+          `[runOfflineFull] explore: storefront tab failed to load (${(err as Error).message}) — continuing`,
+        );
+      }
+      // Leave the admin tab focused as the active one.
+      await explorePage.bringToFront().catch(() => {});
+      console.log(`     Storefront tab opened: ${storefrontUrl}`);
+
       // Wait indefinitely. Exit triggers:
       //   - Developer closes the browser window → context emits `close`.
       //   - Container receives SIGINT/SIGTERM → we close the context
@@ -1361,16 +1383,20 @@ async function main(): Promise<void> {
 
     // ── Playwright ─────────────────────────────────────────
     console.log('[runOfflineFull] running Playwright…');
-    // Opt-in screen recording: run headed under Xvfb :99 so ffmpeg can
-    // x11grab the display (admin + storefront, one real video). Xvfb
-    // ships in the Playwright base image. Start it if the entrypoint
-    // didn't already (idempotent — reuse an existing :99 socket).
+    // Headed-under-Xvfb is the DEFAULT for offline runs (production-
+    // faithful, ~4% slower than headless). Opt OUT with
+    // TEST_OFFLINE_HEADLESS=true. Full-screen recording (TEST_OFFLINE_VIDEO)
+    // and TEST_VISIBLE always force headed. Xvfb ships in the Playwright
+    // base image; ffmpeg (for recording) is baked in too. We start Xvfb
+    // :99 whenever headed, and only require ffmpeg when recording.
     let videoOn = process.env['TEST_OFFLINE_VIDEO'] === 'true';
+    const headlessOn = process.env['TEST_OFFLINE_HEADLESS'] === 'true';
+    let headedOn = videoOn || process.env['TEST_VISIBLE'] === 'true' || !headlessOn;
+    let headedReady = false;
     if (videoOn) {
-      const { existsSync } = await import('node:fs');
-      // ffmpeg (with x11grab) is baked into the VM image. An image built
-      // before video support won't have it — degrade gracefully with a
-      // clear rebuild hint rather than failing every test's recording.
+      // Recording needs ffmpeg (with x11grab). An image built before
+      // video support won't have it — degrade to headed-no-record with
+      // a clear rebuild hint rather than failing every test's recording.
       let hasFfmpeg = false;
       try {
         const { execSync: execSyncCheck } = await import('node:child_process');
@@ -1384,19 +1410,19 @@ async function main(): Promise<void> {
           '[runOfflineFull] TEST_OFFLINE_VIDEO set but `ffmpeg` is missing from the ' +
             'VM image — rebuild it (it predates video support): ' +
             '`container image rm essential-apps/shopify-test-vm:latest` then re-run. ' +
-            'Continuing WITHOUT video.',
+            'Continuing headed WITHOUT recording.',
         );
         videoOn = false;
         process.env['TEST_OFFLINE_VIDEO'] = 'false';
       }
     }
-    if (videoOn) {
+    if (headedOn) {
       const { existsSync } = await import('node:fs');
       if (!existsSync('/tmp/.X11-unix/X99')) {
         const { spawn: spawnXvfb } = await import('node:child_process');
         const xv = spawnXvfb(
           'Xvfb',
-          [':99', '-screen', '0', '1280x720x24', '-nolisten', 'tcp', '-ac'],
+          [':99', '-screen', '0', '1400x900x24', '-nolisten', 'tcp', '-ac'],
           { detached: true, stdio: 'ignore' },
         );
         xv.unref();
@@ -1405,17 +1431,26 @@ async function main(): Promise<void> {
           await new Promise((r) => setTimeout(r, 100));
         }
       }
-      if (!existsSync('/tmp/.X11-unix/X99')) {
-        console.error('[runOfflineFull] WARNING: Xvfb :99 did not start — video disabled');
+      headedReady = existsSync('/tmp/.X11-unix/X99');
+      if (!headedReady) {
+        console.error(
+          '[runOfflineFull] WARNING: Xvfb :99 did not start — falling back to headless',
+        );
+        videoOn = false;
+        headedOn = false;
+        process.env['TEST_OFFLINE_VIDEO'] = 'false';
+        process.env['TEST_OFFLINE_HEADED'] = 'false';
       } else {
-        console.error('[runOfflineFull] screen video ON — Xvfb :99 ready (headed)');
+        console.error(
+          `[runOfflineFull] headed under Xvfb :99 (${videoOn ? 'recording' : 'no recording'})`,
+        );
       }
     }
 
     const playwrightEnv: NodeJS.ProcessEnv = {
       ...process.env,
       // Point headed Chromium + the screen-video fixture at the Xvfb display.
-      ...(videoOn ? { DISPLAY: ':99' } : {}),
+      ...(headedReady ? { DISPLAY: ':99' } : {}),
       // Expose the per-run test DB to specs so they can seed/inspect the
       // app's OWN Postgres directly (e.g. seed Article rows for
       // list/detail flows). The mock ShopState covers Shopify resources,

package/src/scripts/verifyContracts.ts

@@ -29,8 +29,8 @@
  *
  * Run: `npm run test:online:verify-contracts`.
  */
-import { readdirSync, readFileSync, statSync } from 'node:fs';
-import { resolve, relative } from 'node:path';
+import { resolve } from 'node:path';
+import { resolveContracts } from '@essential-apps/shopify-test-contracts/operation-contract';
 import {
   createAdminApi,
   createStorefrontApi,
@@ -112,16 +112,6 @@ function buildOfflineExecutor(api: ApiType, state: ShopState): Executor {
   };
 }
 
-interface Contract {
-  operationName: string;
-  source: string;
-  variables: Record<string, unknown>;
-  response: unknown;
-  capturedFrom: 'offline' | 'live';
-  capturedAt: string;
-  warning?: string;
-}
-
 /**
  * Same deterministic ShopState seed `captureContracts` uses. The
  * contracts were captured against this state; verification must
@@ -214,21 +204,24 @@ function diff(
 async function main(): Promise<void> {
   const args = parseArgs();
 
-  let entries: string[];
-  try {
-    entries = readdirSync(args.contractsDir);
-  } catch {
+  // Merged view: shared package goldens (the centralized generic ops)
+  // overlaid by the app's local contracts dir, which overrides by
+  // operation name. An app vendors a local file only when its shape
+  // genuinely diverges from the shared golden.
+  const resolved = resolveContracts({ api: args.api, appDir: args.contractsDir });
+  if (resolved.length === 0) {
     console.error(
-      `[verify-contracts] no contracts directory at ${args.contractsDir}. ` +
-        `Run \`npm run test:online:capture-contracts\` first.`,
+      `[verify-contracts] no contracts for api=${args.api} ` +
+        `(shared package + ${args.contractsDir}).`,
     );
     process.exit(2);
   }
-  const contractFiles = entries.filter((f) => f.endsWith('.json'));
-  if (contractFiles.length === 0) {
-    console.error(`[verify-contracts] no contracts found in ${args.contractsDir}`);
-    process.exit(2);
-  }
+  const sharedCount = resolved.filter((r) => r.origin === 'shared').length;
+  const appCount = resolved.length - sharedCount;
+  console.log(
+    `[verify-contracts] api=${args.api}: ${resolved.length} operations ` +
+      `(${sharedCount} shared, ${appCount} app-local override)`,
+  );
 
   const state = buildSeededState();
   const executor: Executor =
@@ -239,16 +232,9 @@ async function main(): Promise<void> {
   let skipped = 0;
   const failures: { contract: string; diff: string }[] = [];
 
-  for (const f of contractFiles) {
-    const path = resolve(args.contractsDir, f);
-    let st;
-    try {
-      st = statSync(path);
-    } catch {
-      continue;
-    }
-    if (!st.isFile()) continue;
-    const contract = JSON.parse(readFileSync(path, 'utf8')) as Contract;
+  for (const entry of resolved) {
+    const contract = entry.contract;
+    const label = `${entry.operationName} [${entry.origin}]`;
     // Contracts captured with a warning never executed cleanly;
     // skip them in verify (they need fixtures.json before they can
     // be verified).
@@ -262,7 +248,7 @@ async function main(): Promise<void> {
     } catch (err) {
       drift++;
       failures.push({
-        contract: f,
+        contract: label,
         diff: `executor threw: ${(err as Error).message}`,
       });
       continue;
@@ -280,7 +266,7 @@ async function main(): Promise<void> {
       pass++;
     } else {
       drift++;
-      failures.push({ contract: f, diff: d });
+      failures.push({ contract: label, diff: d });
     }
   }
 
@@ -291,9 +277,7 @@ async function main(): Promise<void> {
     console.log('');
     console.log('[verify-contracts] drift:');
     for (const f of failures) {
-      console.log(
-        `  ${relative(args.cwd, resolve(args.contractsDir, f.contract))}`,
-      );
+      console.log(`  ${f.contract}`);
       console.log(`    ${f.diff}`);
     }
     console.log('');