@essential-apps/shopify-test-runner 1.0.0

package/src/probes/runProbe.ts

@@ -0,0 +1,257 @@
+#!/usr/bin/env node
+/**
+ * Probe runner — entry point for the online-store inspection framework.
+ *
+ * Invocation:
+ *   node --import tsx \
+ *     node_modules/@essential-apps/shopify-test-runner/src/probes/runProbe.ts \
+ *     <probe-name> [--key=value …]
+ *
+ * What it does:
+ *   1. Loads the consuming app's `.env.test` so probes can read
+ *      env (e.g. STOREFRONT_PASSWORD).
+ *   2. Picks an installed store from `tests/test-online/.stores.json`.
+ *   3. Launches Playwright chromium via patchright with the
+ *      saved Partner storage state — same recipe online tests use.
+ *   4. Hands control to the probe.
+ *
+ * Adding a probe: drop a file in `packages/runner/src/probes/<name>.ts`
+ * default-exporting a {@link Probe}, and add a row to `PROBES` below.
+ */
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { type BrowserContext } from '@playwright/test';
+import {
+  readRegistry,
+  storageStatePath,
+  type Store,
+} from '@essential-apps/shopify-test-core';
+import type { Probe, ProbeContext } from './types.js';
+import fontsProbe from './fonts.js';
+import mirrorProbe from './mirror.js';
+import { launchStealthBrowser } from '../lib/stealthLaunch.js';
+
+// ── Registry of available probes ──────────────────────────────
+// Order matters only for `--help` listing.
+const PROBES: Record<string, Probe> = {
+  [fontsProbe.name]: fontsProbe,
+  [mirrorProbe.name]: mirrorProbe,
+};
+
+// ── CLI arg parsing ───────────────────────────────────────────
+
+interface ParsedArgs {
+  name: string;
+  args: Record<string, string>;
+  help: boolean;
+}
+
+function parseArgs(argv: string[]): ParsedArgs {
+  const [name, ...rest] = argv;
+  const args: Record<string, string> = {};
+  let help = false;
+  for (const a of rest) {
+    if (a === '--help' || a === '-h') {
+      help = true;
+      continue;
+    }
+    const m = a.match(/^--([^=]+)=(.*)$/);
+    if (m) {
+      args[m[1]!] = m[2]!;
+      continue;
+    }
+    // Bare flag: --foo (no value) → true
+    const bare = a.match(/^--(.+)$/);
+    if (bare) {
+      args[bare[1]!] = '';
+    }
+  }
+  return { name: name ?? '', args, help };
+}
+
+function printUsage(): void {
+  console.error('Usage: runProbe <name> [--key=value …]');
+  console.error('');
+  console.error('Available probes:');
+  for (const p of Object.values(PROBES)) {
+    console.error(`  ${p.name.padEnd(14)} ${p.description}`);
+  }
+  console.error('');
+  console.error('Global flags:');
+  console.error('  --help                    Show this help');
+  console.error('  --store=<slug>            Use a specific store from the pool (default: any available)');
+  console.error('');
+}
+
+// ── Workspace path discovery ──────────────────────────────────
+
+/**
+ * Locate the workspace root. When this script runs from a packed
+ * tarball inside a consumer's node_modules, the workspace is the
+ * shopify-test repo's root — the package nearest us with
+ * `name === "@essential-apps/shopify-test"`. When running from the
+ * repo itself, that's the repo root.
+ *
+ * We walk up from this file looking for a `package.json` whose
+ * `name` matches, falling back to the nearest directory containing
+ * a `packages/` folder (matches the monorepo layout).
+ */
+function findWorkspaceRoot(): string {
+  // When invoked from a consumer, the runner's source lives under
+  // `node_modules/@essential-apps/shopify-test-runner/src/`. That
+  // path doesn't help us reach the workspace's themes/, etc.
+  // Look upward from this file for the workspace root marker.
+  let dir = dirname(fileURLToPath(import.meta.url));
+  for (let i = 0; i < 10; i++) {
+    const pj = resolve(dir, 'package.json');
+    if (existsSync(pj)) {
+      try {
+        const pkg = JSON.parse(readFileSync(pj, 'utf8')) as { name?: string };
+        if (pkg.name === '@essential-apps/shopify-test') return dir;
+      } catch {
+        /* fall through */
+      }
+    }
+    const parent = resolve(dir, '..');
+    if (parent === dir) break;
+    dir = parent;
+  }
+  // Fallback: assume cwd is the consumer app — outputs go to its
+  // tests/test-online/probe-output/ if we can't find a workspace.
+  return process.cwd();
+}
+
+// ── Store selection ───────────────────────────────────────────
+
+function pickStore(slug?: string): Store {
+  const { stores } = readRegistry();
+  if (slug) {
+    const s = stores.find((x) => x.slug === slug);
+    if (!s) {
+      throw new Error(
+        `Store with slug "${slug}" not found in registry. ` +
+          `Available: ${stores.map((x) => x.slug).join(', ')}`,
+      );
+    }
+    return s;
+  }
+  const candidate =
+    stores.find((s) => s.appInstalled && s.status === 'available') ??
+    stores.find((s) => s.appInstalled) ??
+    stores[0];
+  if (!candidate) {
+    throw new Error(
+      'No stores in registry. Run `npm run test:online:add` + `npm run test:online:install` first.',
+    );
+  }
+  return candidate;
+}
+
+/**
+ * Synthesize a Store record from a bare URL. Used by `--url=<full>`
+ * mode so probes can run against arbitrary public Shopify storefronts
+ * (e.g. `theme-dawn-demo.myshopify.com` for canonical Dawn font
+ * mirroring) without needing a Partner-auth registry entry.
+ */
+function storeFromUrl(rawUrl: string): Store {
+  const u = new URL(rawUrl);
+  const shop = u.hostname;
+  const slug = shop.replace(/\.myshopify\.com$/, '');
+  return {
+    shop,
+    slug,
+    plan: 'PUBLIC',
+    appInstalled: false,
+    status: 'available',
+  };
+}
+
+// ── Browser bootstrap ─────────────────────────────────────────
+
+async function launchBrowser(opts: { needAuth: boolean }): Promise<BrowserContext> {
+  // Reuse the same chrome+patchright recipe as the online-test store
+  // pool: real Google Chrome via patchright. For probes against
+  // Partner-owned stores we layer in the saved Partner storage
+  // state so password gates / admin redirects work; for public
+  // storefronts (--url mode) we skip that — no auth needed.
+  const browser = await launchStealthBrowser({ headless: true });
+  const storageState =
+    opts.needAuth && existsSync(storageStatePath)
+      ? JSON.parse(readFileSync(storageStatePath, 'utf8'))
+      : undefined;
+  if (opts.needAuth && !storageState) {
+    throw new Error(
+      `Storage state not found at ${storageStatePath}. ` +
+        `Run online tests at least once to capture Partner auth cookies, ` +
+        `or pass --url=<public-store-url> to skip auth.`,
+    );
+  }
+  const context = await browser.newContext({
+    viewport: { width: 1400, height: 900 },
+    ignoreHTTPSErrors: true,
+    ...(storageState ? { storageState } : {}),
+  });
+  return context;
+}
+
+// ── Main ──────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  const parsed = parseArgs(process.argv.slice(2));
+  if (parsed.help || !parsed.name) {
+    printUsage();
+    process.exit(parsed.name ? 0 : 1);
+  }
+
+  const probe = PROBES[parsed.name];
+  if (!probe) {
+    console.error(`Unknown probe: "${parsed.name}"\n`);
+    printUsage();
+    process.exit(1);
+  }
+
+  const workspaceRoot = findWorkspaceRoot();
+  // --url=<full> mode: synthesize a Store record from the URL and
+  // skip auth. For probing arbitrary public storefronts (e.g.
+  // theme-dawn-demo.myshopify.com for canonical Dawn font mirroring).
+  const urlMode = parsed.args['url'];
+  const store = urlMode ? storeFromUrl(urlMode) : pickStore(parsed.args['store']);
+  const outputDir = resolve(
+    workspaceRoot,
+    probe.outputDir.replace(/\$\{name\}/g, probe.name),
+  );
+  mkdirSync(outputDir, { recursive: true });
+
+  console.log(`[runProbe] probe: ${probe.name}`);
+  console.log(`[runProbe] store: ${store.shop} (${store.slug})${urlMode ? ' [public --url mode]' : ''}`);
+  console.log(`[runProbe] output: ${outputDir}`);
+
+  const browserContext = await launchBrowser({ needAuth: !urlMode });
+  const page = await browserContext.newPage();
+
+  const ctx: ProbeContext = {
+    store,
+    browserContext,
+    page,
+    outputDir,
+    args: parsed.args,
+    workspaceRoot,
+  };
+
+  try {
+    await probe.run(ctx);
+    console.log(`[runProbe] ✓ ${probe.name} complete`);
+  } catch (err) {
+    console.error(`[runProbe] ✗ ${probe.name} failed:`, err);
+    process.exitCode = 1;
+  } finally {
+    await browserContext.close().catch(() => {});
+    await browserContext.browser()?.close().catch(() => {});
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/probes/types.ts

@@ -0,0 +1,73 @@
+/**
+ * Probe framework — a reusable harness for asking real Shopify
+ * "what does this actually look like?" against the online test store
+ * pool.
+ *
+ * Why this exists: every offline mock has a long tail of "is our
+ * shape right?" questions. Examples we've already hit:
+ *   - what URLs does Shopify's `font_face` Liquid filter emit?
+ *   - what does the real `themes/{id}/assets.json` response look like?
+ *   - what HTTP headers does fonts.shopifycdn.com set on a .woff2?
+ *   - what's the exact `settings_data.json` shape when an app embed
+ *     is activated?
+ *
+ * Each probe is a small Node script that drives a Playwright browser
+ * against a real online store (picked from `tests/test-online/.stores.json`),
+ * captures structured fixture data, and writes it somewhere the
+ * mocks can read at runtime.
+ *
+ * Adding a probe: create `packages/runner/src/probes/<name>.ts`,
+ * default-export an object conforming to {@link Probe}, and add a
+ * row to the registry in `runProbe.ts`.
+ *
+ * Invoke from the consuming app's repo:
+ *   node --import tsx \
+ *     node_modules/@essential-apps/shopify-test-runner/src/probes/runProbe.ts \
+ *     <probe-name> [--key=value …]
+ */
+import type { BrowserContext, Page } from '@playwright/test';
+import type { Store } from '@essential-apps/shopify-test-core';
+
+/**
+ * Per-probe execution context. The runner sets up the browser +
+ * store before invoking the probe; the probe focuses on what it
+ * captures, not how to get a browser.
+ */
+export interface ProbeContext {
+  /** The online store this probe runs against. */
+  store: Store;
+  /** Logged-in Playwright BrowserContext (storage state pre-loaded). */
+  browserContext: BrowserContext;
+  /** Convenience: a fresh page in the context. Probes can open more. */
+  page: Page;
+  /**
+   * Absolute path where probe artifacts go. Probes write fixture
+   * files (JSON, binary, etc.) under here so the mock packages
+   * can read them later. Conventionally the workspace path
+   * `packages/<owner-package>/fixtures/<probe-name>/`.
+   */
+  outputDir: string;
+  /** Parsed --key=value CLI args. */
+  args: Record<string, string>;
+  /** Workspace root absolute path (the shopify-test repo). */
+  workspaceRoot: string;
+}
+
+export interface Probe {
+  /** Probe name — what you type after `runProbe`. */
+  name: string;
+  /** One-line description shown in `runProbe --help`. */
+  description: string;
+  /**
+   * Where outputs should go, relative to workspaceRoot. The harness
+   * creates this dir and passes its absolute path as
+   * `ctx.outputDir`. Use `${name}` etc. — substituted at runtime.
+   */
+  outputDir: string;
+  /**
+   * Main entry point. Throws on failure; logs progress to stdout.
+   * Should be idempotent — re-running a probe should refresh
+   * outputs cleanly.
+   */
+  run(ctx: ProbeContext): Promise<void>;
+}

package/src/scripts/addStore.ts

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+import { parseArgs } from 'node:util';
+import {
+  loadEnv,
+  printEnvSummary,
+  appendStore,
+  readRegistry,
+} from '@essential-apps/shopify-test-core';
+
+async function main(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      shop: { type: 'string' },
+      plan: { type: 'string' },
+    },
+  });
+
+  const shop = values.shop?.trim();
+  if (!shop) {
+    console.error(
+      'Usage: npm run test:online:add -- --shop online-1.myshopify.com [--plan UNLIMITED_APP_DEVELOPMENT]',
+    );
+    process.exit(1);
+  }
+  if (!/^[a-z0-9-]+\.myshopify\.com$/.test(shop)) {
+    console.error(`Invalid shop format. Expected "<slug>.myshopify.com", got "${shop}".`);
+    process.exit(1);
+  }
+
+  const env = loadEnv();
+  printEnvSummary(env);
+
+  const reg = readRegistry();
+  if (reg.stores.some((s) => s.shop === shop)) {
+    console.error(`Already in registry: ${shop}`);
+    process.exit(1);
+  }
+
+  const slug = shop.replace('.myshopify.com', '');
+  appendStore({
+    shop,
+    slug,
+    plan: values.plan ?? env.plan,
+    createdAt: new Date().toISOString(),
+    appInstalled: false,
+    status: 'available',
+  });
+
+  console.log(`✓ Added ${shop} to pool.`);
+  console.log('');
+  console.log('Next: install the dev app on this store.');
+  console.log(`  Make sure your dev server is running, then:`);
+  console.log(`    npm run test:online:install -- --shop ${shop}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/scripts/buildDockerImage.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/**
+ * Build the canonical `shopify-test` image from the Dockerfile
+ * bundled in this package. Run from the shopify-test workspace
+ * root:
+ *
+ *   npm run docker:build
+ *
+ * Or directly:
+ *
+ *   node --import tsx packages/runner/src/scripts/buildDockerImage.ts
+ *
+ * The image is amd64-only (Rosetta on Apple Silicon) — see
+ * docker/README.md for why.
+ */
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// Mirrors the npm scope (`@essential-apps/shopify-test`) so anyone
+// inspecting `container images` sees the same name they `npm install`.
+const DEFAULT_TAG = 'essential-apps/shopify-test:latest';
+
+function dockerContextDir(): string {
+  // This file lives at packages/runner/src/scripts/ (or dist/scripts/
+  // after build). Walk up + sideways to docker/.
+  const here = dirname(fileURLToPath(import.meta.url));
+  // here = .../packages/runner/{src,dist}/scripts
+  const candidates = [
+    resolve(here, '../../docker'),
+    resolve(here, '../docker'),
+  ];
+  for (const c of candidates) {
+    if (existsSync(resolve(c, 'Dockerfile'))) return c;
+  }
+  throw new Error(
+    `Could not locate the Dockerfile under packages/runner/docker/. ` +
+      `Tried: ${candidates.join(', ')}`,
+  );
+}
+
+async function main(): Promise<void> {
+  const tag = process.env['SHOPIFY_TEST_IMAGE_TAG'] ?? DEFAULT_TAG;
+  const context = dockerContextDir();
+
+  console.log('────────────────────────────────────────────────────────────');
+  console.log(' shopify-test docker image build');
+  console.log('────────────────────────────────────────────────────────────');
+  console.log(`  Tag      : ${tag}`);
+  console.log(`  Context  : ${context}`);
+  console.log(`  Arch     : amd64`);
+  console.log('────────────────────────────────────────────────────────────');
+  console.log('');
+
+  const args = ['build', '--arch', 'amd64', '--tag', tag, context];
+  const p = spawn('container', args, { stdio: 'inherit' });
+  p.on('exit', (code) => process.exit(code ?? 0));
+  process.on('SIGINT', () => p.kill('SIGINT'));
+  process.on('SIGTERM', () => p.kill('SIGTERM'));
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/src/scripts/captureAuth.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/**
+ * Capture Shopify auth state into tests/test-online/.auth/storageState.json.
+ *
+ * The captured cookies are fingerprint-bound (UA, sec-ch-ua-platform,
+ * canvas/WebGL hashes, etc.), so this script MUST run in the same
+ * browser environment that tests use:
+ *   - On host (macOS): Google Chrome via patchright.
+ *   - In container: Google Chrome via patchright (same channel),
+ *     under Xvfb. The runDockerAuth.ts wrapper exposes the Xvfb
+ *     display over VNC so the user can interactively log in.
+ *
+ * If you re-capture on host but tests run in container (or vice-versa),
+ * Cloudflare's bot heuristics will see UA / fingerprint inconsistency
+ * and challenge or block the session.
+ */
+import { mkdirSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname } from 'node:path';
+import { createInterface } from 'node:readline';
+import type { BrowserContext, chromium as playwrightChromium } from '@playwright/test';
+import { storageStatePath, assertInVm } from '@essential-apps/shopify-test-core';
+
+// Auth capture must run INSIDE the VM (via runVmAuth, which sets
+// TEST_IN_CONTAINER + drives this over VNC) — never on the host, or the
+// captured cf_clearance/session is fingerprint-mismatched against the VM.
+assertInVm('capture Shopify auth');
+
+const require = createRequire(import.meta.url);
+const chromium = (require('patchright') as { chromium: typeof playwrightChromium }).chromium;
+
+// Test Partner account (throwaway; not prod). Shown in the terminal AND
+// pinned to a bar at the top of the browser so it's readable while
+// logging in over VNC.
+const LOGIN_EMAIL = 'essential-apps-test-1@supercorp.ai';
+const LOGIN_PASSWORD = 'essential-apps-test-1-password';
+
+// A fixed top bar injected into every page (survives navigation) showing
+// the login creds. `pointer-events:none` so it never blocks the form;
+// re-pinned on an interval because Shopify's accounts pages re-render.
+const credsBarInitScript = `(() => {
+  const ID = '__ea_auth_creds_bar__';
+  const text = ${JSON.stringify(`TEST LOGIN  ·  ${LOGIN_EMAIL}  ·  password: ${LOGIN_PASSWORD}`)};
+  const render = () => {
+    if (!document.documentElement) return;
+    let b = document.getElementById(ID);
+    if (!b) {
+      b = document.createElement('div');
+      b.id = ID;
+      b.style.cssText = 'position:fixed;top:0;left:0;right:0;z-index:2147483647;background:#0a7d3b;color:#fff;font:600 13px/26px ui-monospace,SFMono-Regular,Menlo,monospace;text-align:center;height:26px;padding:0 10px;box-shadow:0 1px 4px rgba(0,0,0,.35);pointer-events:none';
+      (document.body || document.documentElement).appendChild(b);
+    }
+    b.textContent = text;
+  };
+  render();
+  document.addEventListener('DOMContentLoaded', render);
+  setInterval(render, 800);
+})();`;
+
+async function main(): Promise<void> {
+  console.log('Opening Chrome. Please:');
+  console.log(`  Log in as:  ${LOGIN_EMAIL}`);
+  console.log(`  Password:   ${LOGIN_PASSWORD}   (also in the green bar atop the browser)`);
+  console.log('  1. Log in to Shopify Partners.');
+  console.log('  2. Then visit your test store admin (e.g. https://admin.shopify.com/store/<your-store>) —');
+  console.log('     if a Cloudflare "Verify you are human" challenge appears, click through it.');
+  console.log('     Without this step, tests will hit Turnstile and fail.');
+  console.log('  3. When both are done, return here and press Enter.');
+  console.log('');
+
+  // chromium.launch + newContext (matches storePool fixture model).
+  // Critically, this is NOT launchPersistentContext — the persistent
+  // profile accumulates fingerprint state that triggers Cloudflare's
+  // bot detection. Fresh-context model means the browser fingerprint
+  // CF sees here (during capture) is the SAME fingerprint it'll see
+  // at test time (when a fresh context is built from this saved
+  // storageState). cf_clearance is fingerprint-bound, so this
+  // consistency is what makes the saved cookie actually work.
+  //
+  // Patchright (no `channel: 'chrome'`) — uses its bundled
+  // chromium with stealth patches. Same browser as storePool +
+  // conformance probes, so cf_clearance carries across all three.
+  // Real Google Chrome was tried earlier but its fingerprint
+  // doesn't match patchright's, causing CF re-challenges.
+  const browser = await chromium.launch({
+    headless: false,
+    args: [
+      '--ip-address-space-overrides=127.0.0.1:0=public,[::1]:0=public',
+      '--disable-features=LocalNetworkAccessChecks,LocalNetworkAccessChecksWebSockets,LocalNetworkAccessChecksWebTransport,BlockInsecurePrivateNetworkRequests,PrivateNetworkAccessRespectPreflightResults,PrivateNetworkAccessSendPreflights',
+      '--local-network-access-permissions-policy-default-enabled',
+    ],
+  });
+  const context = (await browser.newContext({
+    viewport: { width: 1400, height: 900 },
+    ignoreHTTPSErrors: true,
+  })) as unknown as BrowserContext;
+
+  // Pin the creds to a bar at the top of the (VNC'd) browser.
+  await context.addInitScript(credsBarInitScript);
+
+  const page = await context.newPage();
+  await page.goto('https://accounts.shopify.com/lookup');
+
+  await waitForEnter('Press Enter once logged in AND past any admin Turnstile... ');
+
+  // Sanity-check cf_clearance presence and warn loudly if absent —
+  // capture without it works for Partner-API-only flows, but online
+  // tests that drive admin.shopify.com will fail on Turnstile.
+  const cookies = await context.cookies();
+  const hasCfClearance = cookies.some(
+    (c) => c.name === 'cf_clearance' && c.domain.includes('shopify.com'),
+  );
+  if (!hasCfClearance) {
+    console.warn(
+      '⚠️  No cf_clearance cookie found for *.shopify.com. The online suite will hit Turnstile.',
+    );
+    console.warn(
+      '   Visit your test store admin in this browser, pass the Cloudflare challenge, then re-run.',
+    );
+  } else {
+    console.log('✓ cf_clearance cookie captured (admin.shopify.com Turnstile bypass).');
+  }
+
+  mkdirSync(dirname(storageStatePath), { recursive: true });
+  await context.storageState({ path: storageStatePath });
+  console.log(`✓ Auth state saved: ${storageStatePath}`);
+
+  await context.close();
+  await browser.close();
+}
+
+function waitForEnter(prompt: string): Promise<void> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise((res) =>
+    rl.question(prompt, () => {
+      rl.close();
+      res();
+    }),
+  );
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});