@essential-apps/shopify-test-runner 1.0.0

package/src/lib/sourceZipUpload.ts

@@ -0,0 +1,168 @@
+/**
+ * Upload an app-source zip to Shopify's signed-URL bucket, so we can
+ * reference it via `AppVersionInput.sourceUrl` in `appVersionCreate`.
+ *
+ * Why we need this
+ * ----------------
+ * Theme app extensions can be deployed via `appVersionCreate(version:
+ * { source: <manifest with files inline as base64> })` — the file
+ * contents fit comfortably in the GraphQL request body. Function
+ * (WASM) and post-purchase (JS bundle) extensions, however, are
+ * uploaded out-of-band:
+ *
+ *   1. Call `appRequestSourceUploadUrl(sourceExtension: ZIP, …)` —
+ *      Shopify returns a 1-hour GCS v4-signed PUT URL.
+ *   2. PUT the zip bytes to that URL.
+ *   3. Call `appVersionCreate(version: { sourceUrl: <same URL> })`.
+ *
+ * The CLI does the same — see
+ * `Shopify/cli@packages/app/src/cli/services/bundle.ts` (`getUploadURL`
+ * + `uploadToGCS`). We confirmed empirically that:
+ *   - The mutation accepts `sourceExtension: ZIP | BR`
+ *     (we only emit ZIP).
+ *   - The returned `sourceUploadUrl` is a GCS v4 presigned URL with
+ *     `X-Goog-Algorithm=GOOG4-RSA-SHA256` and `X-Goog-Expires=3600`.
+ *   - `appVersionCreate` validates the URL against an allowlist —
+ *     URLs from other hosts are rejected with HTTP 500
+ *     `"URL path is not valid"`. So you can ONLY use a sourceUrl
+ *     minted by `appRequestSourceUploadUrl`.
+ *
+ * Quirks
+ * ------
+ * - The CLI ostensibly builds a multipart form to derive the upload
+ *   headers, then sends the raw buffer as body. Empirically, a plain
+ *   PUT with `Content-Type: application/zip` works against the GCS
+ *   v4 presigned URL — that's what we do.
+ * - There's no MAX_SIZE error message in the API response, but the
+ *   CLI hard-codes a 100 MB cap. Our zips are far below that.
+ */
+import { createReadStream, statSync } from 'node:fs';
+import { Readable } from 'node:stream';
+import { appManagementGraphQL } from '@essential-apps/shopify-test-core';
+
+/**
+ * GraphQL mutation to mint a signed upload URL.
+ *
+ * Defaults to `sourceExtension: BR` (tar+brotli archive) — that's the
+ * format that actually works for full-fidelity deploys including
+ * functions. The legacy `ZIP` option exists in the schema and returns
+ * a working signed URL, but the server 500s on zips containing per-
+ * extension folder layouts. See lib/buildSourceBundle.ts.
+ *
+ * NOTE on schema discovery: schema introspection against the
+ * `/unstable/` endpoint is rejected with `404 "Cannot find a valid
+ * organization"` for tokens minted via App-automation-token exchange
+ * — only targeted queries scoped to an app/org work. Mutation name +
+ * arg shape were confirmed empirically.
+ */
+const REQUEST_SOURCE_UPLOAD_URL_MUTATION = `
+  mutation RequestSourceUploadUrl(
+    $organizationId: ID!
+    $sourceExtension: SourceExtension!
+  ) {
+    appRequestSourceUploadUrl(
+      sourceExtension: $sourceExtension
+      organizationId: $organizationId
+    ) {
+      sourceUploadUrl
+      userErrors { field message }
+    }
+  }
+`;
+
+export interface RequestSourceUploadUrlOptions {
+  /** Bearer token from `exchangeAutomationToken`. */
+  accessToken: string;
+  /** Numeric Partner-org ID. Required — the endpoint won't infer it
+   *  from the access token, even though it's already scoped to one
+   *  org. (Discovered empirically: omitting it yields a generic 404.) */
+  organizationId: string;
+  /**
+   * Archive format. Defaults to 'BR' (tar+brotli) which the
+   * App Management server fully supports including function
+   * extensions. 'ZIP' is accepted for legacy callers but the
+   * server's function-deploy path 500s on zip-with-folders.
+   */
+  sourceExtension?: 'BR' | 'ZIP';
+}
+
+/**
+ * Mint a fresh signed upload URL. The URL is valid for 3600 seconds;
+ * upload + version-create should both happen well within that window.
+ *
+ * Returns the GCS URL verbatim — you pass the SAME URL to
+ * `appVersionCreate` as `sourceUrl`. Shopify maintains the mapping
+ * server-side between the signed URL and the uploaded object.
+ */
+export async function requestSourceUploadUrl(
+  opts: RequestSourceUploadUrlOptions,
+): Promise<string> {
+  const data = await appManagementGraphQL<{
+    appRequestSourceUploadUrl: {
+      sourceUploadUrl?: string;
+      userErrors: Array<{ field?: string[]; message: string }>;
+    };
+  }>(opts.accessToken, REQUEST_SOURCE_UPLOAD_URL_MUTATION, {
+    organizationId: `gid://shopify/Organization/${opts.organizationId}`,
+    sourceExtension: opts.sourceExtension ?? 'BR',
+  });
+  const errs = data.appRequestSourceUploadUrl.userErrors;
+  if (errs.length > 0) {
+    throw new Error(
+      'appRequestSourceUploadUrl userErrors:\n' +
+        errs.map((e) => `  • ${(e.field ?? []).join('.')}: ${e.message}`).join('\n'),
+    );
+  }
+  const url = data.appRequestSourceUploadUrl.sourceUploadUrl;
+  if (!url) {
+    throw new Error('appRequestSourceUploadUrl returned no sourceUploadUrl');
+  }
+  return url;
+}
+
+export interface UploadSourceOptions {
+  uploadUrl: string;
+  /** Path on disk to the archive file (typically a `.tar.br` from
+   *  `buildSourceBundle`, or a legacy `.zip`). */
+  archivePath: string;
+}
+
+/**
+ * PUT an archive file to the signed GCS URL minted by
+ * `requestSourceUploadUrl`. The URL embeds its own auth (signed
+ * `X-Goog-*` query params), so no bearer is needed on this request.
+ *
+ * Headers: we send only `Content-Length`. The CLI doesn't set a
+ * Content-Type for these uploads (it builds a multipart form to
+ * derive headers but sends the raw buffer as body, and GCS ignores
+ * the content-type anyway because the signed URL pins everything
+ * server-side). We match that for parity.
+ *
+ * Streams the file so a multi-MB archive doesn't sit in memory.
+ */
+export async function uploadSourceToSignedUrl(
+  opts: UploadSourceOptions,
+): Promise<void> {
+  const size = statSync(opts.archivePath).size;
+  const stream = createReadStream(opts.archivePath);
+  // Convert Node Readable -> Web ReadableStream for the global
+  // `fetch` API. Node 18+ exposes `Readable.toWeb`.
+  const body = Readable.toWeb(stream) as unknown as ReadableStream<Uint8Array>;
+  const r = await fetch(opts.uploadUrl, {
+    method: 'PUT',
+    body,
+    headers: { 'Content-Length': String(size) },
+    // Node's fetch needs duplex when streaming a body.
+    // @ts-expect-error — `duplex` is a recent option not yet in some
+    // @types/node releases.
+    duplex: 'half',
+  });
+  if (!r.ok) {
+    const text = await r.text().catch(() => '<no body>');
+    throw new Error(
+      `Source upload to signed URL failed: HTTP ${r.status} ${r.statusText}\n` +
+        `  ${text.slice(0, 400)}`,
+    );
+  }
+}
+

package/src/lib/stealthLaunch.ts

@@ -0,0 +1,57 @@
+import { createRequire } from 'node:module';
+import type { Browser } from '@playwright/test';
+import { assertInVm } from '@essential-apps/shopify-test-core';
+
+// patchright is published CJS-only; this file is ESM (tsx/Node).
+const require = createRequire(import.meta.url);
+
+/**
+ * Canonical Cloudflare / Private-Network-Access bypass flags.
+ *
+ * Kept byte-identical to the admin `storePool` fixture's launch args:
+ * cf_clearance binds to the (engine, UA, flags) fingerprint, so any
+ * browser that navigates the real Shopify admin — installApp, the
+ * globalSetup auth pre-flight — MUST launch the same way the online
+ * tests do, or a perfectly good cf_clearance trips Turnstile.
+ */
+export const stealthLaunchArgs = [
+  '--ip-address-space-overrides=127.0.0.1:0=public,[::1]:0=public',
+  '--disable-features=LocalNetworkAccessChecks,LocalNetworkAccessChecksWebSockets,LocalNetworkAccessChecksWebTransport,BlockInsecurePrivateNetworkRequests,PrivateNetworkAccessRespectPreflightResults,PrivateNetworkAccessSendPreflights',
+  '--local-network-access-permissions-policy-default-enabled',
+];
+
+/**
+ * Launch patchright Chromium — the ONE stealth engine used across the
+ * whole suite (online `storePool` fixture, `captureAuth`, every
+ * conformance probe). patchright patches the `Runtime.enable` CDP leak
+ * Cloudflare's bot detection keys on, so its bundled Chromium passes
+ * Turnstile headless — no StealthPlugin, no real-Chrome channel.
+ *
+ * This replaces the old playwright-extra + puppeteer-extra-plugin-stealth
+ * launcher (real Chrome via `channel: 'chrome'`), which was the lone
+ * outlier engine. Standalone scripts (installApp) and the globalSetup
+ * auth pre-flight call this; @playwright/test-managed tests get their
+ * browser from the `storePool` fixture, which launches patchright the
+ * same way.
+ */
+export async function launchStealthBrowser(opts: {
+  headless: boolean;
+  /**
+   * Park the window far offscreen (headed-but-hidden on host) — mirrors
+   * storePool's behaviour for headless:false-on-host launches. Ignored
+   * under Xvfb in-container (no real display there).
+   */
+  hideWindow?: boolean;
+}): Promise<Browser> {
+  assertInVm('launch a stealth browser');
+  const { chromium } = require('patchright') as typeof import('@playwright/test');
+  return (await chromium.launch({
+    headless: opts.headless,
+    args: [
+      ...stealthLaunchArgs,
+      ...(opts.hideWindow
+        ? ['--window-position=-3000,-3000', '--window-size=1400,900']
+        : []),
+    ],
+  })) as unknown as Browser;
+}

package/src/lib/storeAutomation.ts

@@ -0,0 +1,110 @@
+import { existsSync } from 'node:fs';
+import { type BrowserContext, type Page } from '@playwright/test';
+import { storageStatePath } from '@essential-apps/shopify-test-core';
+import { launchStealthBrowser } from './stealthLaunch.js';
+
+export async function launchContext(opts: {
+  headless: boolean;
+  useStorageState: boolean;
+}): Promise<BrowserContext> {
+  if (opts.useStorageState && !existsSync(storageStatePath)) {
+    throw new Error(
+      `Auth state not found at ${storageStatePath}.\n` + `Run: npm run test:online:auth`,
+    );
+  }
+  // patchright (bundled Chromium) via the shared launcher — the one stealth
+  // engine, and the only one that works in the arm64 VM (no real Chrome
+  // there). assertInVm() inside it refuses a host launch.
+  const browser = await launchStealthBrowser({ headless: opts.headless });
+  const context = await browser.newContext({
+    ...(opts.useStorageState ? { storageState: storageStatePath } : {}),
+    viewport: { width: 1400, height: 900 },
+  });
+  await context.addInitScript(() => {
+    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+  });
+  return context;
+}
+
+export function generateStoreName(workerIndex: number): string {
+  return `online-w${workerIndex}-${Date.now()}`;
+}
+
+const PLAN_LABEL: Record<string, string> = {
+  BASIC_APP_DEVELOPMENT: 'Basic',
+  PROFESSIONAL_APP_DEVELOPMENT: 'Professional',
+  UNLIMITED_APP_DEVELOPMENT: 'Unlimited',
+  SHOPIFY_PLUS_APP_DEVELOPMENT: 'Shopify Plus',
+};
+
+/**
+ * Mirrors Shopify CLI's online/setup/store.ts createDevStore flow.
+ * Selectors target stable s-internal-* shadow-DOM hooks that survive
+ * Polaris churn.
+ */
+export async function createDevStore(
+  page: Page,
+  opts: { orgId: string; storeName: string; plan: string },
+): Promise<{ shop: string; slug: string }> {
+  const url = `https://admin.shopify.com/store-create/organization/${opts.orgId}`;
+  await page.goto(url, { waitUntil: 'domcontentloaded' });
+
+  const nameField = page.locator('s-internal-text-field[label="Store name"]').first();
+  await nameField.waitFor({ state: 'visible', timeout: 30_000 });
+  await nameField.locator('input').fill(opts.storeName);
+
+  const planSelect = page.locator('s-internal-select[label="Shopify plan"]').first();
+  await planSelect.click();
+  const planLabel = PLAN_LABEL[opts.plan];
+  if (!planLabel) throw new Error(`Unknown plan: ${opts.plan}`);
+  await page.getByRole('option', { name: new RegExp(planLabel, 'i') }).first().click();
+
+  await page.locator('s-internal-button[variant="primary"]').first().click();
+
+  await page.waitForURL(/admin\.shopify\.com\/store\/[^/]+/, { timeout: 90_000 });
+  const m = page.url().match(/admin\.shopify\.com\/store\/([^/?#]+)/);
+  if (!m) throw new Error(`Unexpected post-create URL: ${page.url()}`);
+  const slug = m[1]!;
+  return { slug, shop: `${slug}.myshopify.com` };
+}
+
+export async function uninstallApp(
+  page: Page,
+  slug: string,
+  appHandle: string,
+): Promise<boolean> {
+  await page.goto(`https://admin.shopify.com/store/${slug}/settings/apps`, {
+    waitUntil: 'domcontentloaded',
+  });
+
+  const row = page.getByRole('link', { name: new RegExp(appHandle, 'i') }).first();
+  if ((await row.count()) === 0) return false;
+
+  await row.click();
+  await page.waitForLoadState('domcontentloaded');
+
+  const uninstallBtn = page.getByRole('button', { name: /uninstall/i }).first();
+  if ((await uninstallBtn.count()) === 0) return false;
+  await uninstallBtn.click();
+
+  const confirmBtn = page.getByRole('button', { name: /uninstall app|confirm|delete/i }).last();
+  await confirmBtn.click();
+  await page.waitForLoadState('networkidle').catch(() => {});
+  return true;
+}
+
+export async function deleteStore(page: Page, slug: string): Promise<void> {
+  await page.goto(`https://admin.shopify.com/store/${slug}/settings/plan/cancel`, {
+    waitUntil: 'domcontentloaded',
+  });
+
+  const checkbox = page.getByRole('checkbox').first();
+  await checkbox.waitFor({ state: 'visible', timeout: 30_000 });
+  await checkbox.check();
+
+  await page.getByRole('button', { name: /delete store/i }).click();
+
+  await page
+    .waitForURL((u: URL) => !u.toString().includes(slug), { timeout: 60_000 })
+    .catch(() => {});
+}

package/src/playwright/baseConfig.ts

@@ -0,0 +1,120 @@
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { defineConfig, type PlaywrightTestConfig } from '@playwright/test';
+import { storageStatePath } from '@essential-apps/shopify-test-core';
+
+/**
+ * Resolve the absolute path to our globalSetup file. Playwright wants
+ * a path string for `globalSetup`; from the consuming app's repo this
+ * is somewhere inside `node_modules/@essential-apps/shopify-test-runner/dist`.
+ *
+ * `.js` (not `.ts`) because this file ships to consumers as built JS.
+ */
+const globalSetupPath = fileURLToPath(new URL('./globalSetup.js', import.meta.url));
+
+export interface DefinePlaywrightConfigOptions {
+  /**
+   * Absolute path to the directory containing the app's `*.spec.ts`
+   * files. Conventionally `tests/test-online/` or `tests/test-offline/`.
+   */
+  testDir: string;
+  /**
+   * Optional: override the Playwright test glob.
+   * Defaults to `**​/*.spec.ts`.
+   */
+  testMatch?: string | string[];
+  /**
+   * Optional: globs to exclude. Defaults to excluding `*offline*.spec.ts`
+   * in online mode (those run via a sibling
+   * `playwright.offline-full.config.ts`), and to `[]` in offline mode
+   * (where testMatch already filters in only the offline specs).
+   */
+  testIgnore?: string | string[];
+  /** Override default workers (default: env TEST_WORKERS or 1). */
+  workers?: number;
+  /** Override default retries (default: env TEST_RETRIES or 1). */
+  retries?: number;
+  /** Per-test timeout in ms. Default 90s. */
+  timeout?: number;
+  /** Per-expect timeout in ms. Default 15s. */
+  expectTimeout?: number;
+  /**
+   * Extra options that will be merged into the returned config —
+   * useful for app-specific reporter tweaks, projects, etc.
+   */
+  override?: PlaywrightTestConfig;
+  /**
+   * Offline-only mode: skip every assertion that exists for online
+   * cloud runs (Shopify Partner storageState, store registry,
+   * NODE_ENV=test, local Postgres). Use this for specs that talk
+   * exclusively to the offline storefront mock — they don't touch
+   * Cloudflare, Shopify auth, or the app backend, so the online
+   * preconditions don't apply.
+   *
+   * Typical setup: dedicated `playwright.config.offline.ts` in the
+   * consuming app, sibling to the online `playwright.config.ts`.
+   */
+  offline?: boolean;
+}
+
+/**
+ * Playwright config preset for Essential Apps' Shopify test suites.
+ *
+ * The consuming app's `tests/test-online/playwright.config.ts` should be a
+ * thin wrapper:
+ *
+ * ```ts
+ * import { definePlaywrightConfig } from '@essential-apps/shopify-test-runner/playwright';
+ * import { fileURLToPath } from 'node:url';
+ *
+ * export default definePlaywrightConfig({
+ *   testDir: fileURLToPath(new URL('.', import.meta.url)),
+ * });
+ * ```
+ */
+export function definePlaywrightConfig(
+  opts: DefinePlaywrightConfigOptions,
+): PlaywrightTestConfig {
+  // Online-mode prerequisites. Offline mode skips the storageState
+  // requirement entirely (offline tests never touch Shopify auth).
+  if (!opts.offline && !existsSync(storageStatePath)) {
+    throw new Error(
+      `Auth state not found at ${storageStatePath}. Run: npm run test:online:auth`,
+    );
+  }
+
+  // In online mode, exclude *offline*.spec.ts by default — those need
+  // the offline runner's distinct fixtures and would either be wasted
+  // (no backend needed) or fail under the online globalSetup. Offline
+  // mode doesn't get a default ignore since its testMatch already
+  // narrows in.
+  const defaultTestIgnore = opts.offline ? [] : ['**/*offline*.spec.ts'];
+  const base: PlaywrightTestConfig = {
+    testDir: opts.testDir,
+    testMatch: opts.testMatch ?? ['**/*.spec.ts'],
+    testIgnore: opts.testIgnore ?? defaultTestIgnore,
+    // No globalSetup in offline mode — its checks (NODE_ENV=test,
+    // local Postgres, store registry with installed apps) are all
+    // online-only invariants.
+    ...(opts.offline ? {} : { globalSetup: globalSetupPath }),
+    timeout: opts.timeout ?? 90_000,
+    expect: { timeout: opts.expectTimeout ?? 15_000 },
+    fullyParallel: false,
+    // Cloudflare flags concurrent contexts from one IP. Bump TEST_WORKERS=2
+    // only after the cf_clearance cookie is established in each worker's
+    // persistent profile (run once with TEST_WORKERS=N to seed profiles).
+    workers: opts.workers ?? Number(process.env['TEST_WORKERS'] ?? 1),
+    // CF's bot challenge can fire on profile creation; one retry is enough.
+    retries: opts.retries ?? Number(process.env['TEST_RETRIES'] ?? 1),
+    reporter: [['list'], ['html', { open: 'never' }]],
+    use: {
+      trace: 'retain-on-failure',
+      screenshot: 'only-on-failure',
+    },
+    // Browser launch is handled by the per-worker `persistentContext`
+    // fixture in @essential-apps/shopify-test-admin (uses
+    // launchPersistentContext so cf_clearance + auth cookies persist).
+  };
+
+  return defineConfig({ ...base, ...opts.override });
+}

package/src/playwright/globalSetup.ts

@@ -0,0 +1,179 @@
+/**
+ * Playwright global setup. Runs ONCE before the suite.
+ *
+ * Hard guards against running tests against non-isolated databases.
+ * The runTests.ts orchestrator sets NODE_ENV=test and DATABASE_URL
+ * pointing at a fresh UUID-named local Postgres DB before invoking
+ * Playwright. We verify both before any test runs.
+ *
+ * Consuming apps reference this from their `playwright.config.ts` via
+ * `definePlaywrightConfig` (which sets `globalSetup` to point at this
+ * file's path).
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { registryPath, storageStatePath } from '@essential-apps/shopify-test-core';
+import { launchStealthBrowser } from '../lib/stealthLaunch.js';
+
+interface RegistryShape {
+  stores: Array<Record<string, unknown>>;
+}
+
+export default async function globalSetup(): Promise<void> {
+  if (!existsSync(registryPath)) {
+    throw new Error(
+      `${registryPath} missing. Run \`npm run test:online:add\` and \`npm run test:online:install\`.`,
+    );
+  }
+  const pool = JSON.parse(readFileSync(registryPath, 'utf8')) as RegistryShape;
+  const installed = pool.stores.filter((s) => s['appInstalled'] === true);
+  if (installed.length === 0) {
+    throw new Error(
+      `No stores in pool have appInstalled: true. Run \`npm run test:online:install\`.`,
+    );
+  }
+
+  const dbUrl = process.env['DATABASE_URL'] ?? '';
+  const nodeEnv = process.env['NODE_ENV'] ?? '';
+
+  if (nodeEnv !== 'test') {
+    throw new Error(
+      `NODE_ENV must be "test" for tests. Got "${nodeEnv}". ` +
+        `Run via \`npm run test:online\` (uses shopify-test-run-tests), not the bare ` +
+        `Playwright entrypoint.`,
+    );
+  }
+
+  if (dbUrl && (dbUrl.includes('neon.tech') || dbUrl.includes('aws.neon'))) {
+    throw new Error(
+      `DATABASE_URL points at Neon (${dbUrl.replace(/:[^:@]+@/, ':***@')}). ` +
+        `Tests must use a local Postgres DB for isolation. Run via \`npm run test:online\`.`,
+    );
+  }
+
+  if (!dbUrl.includes('localhost') && !dbUrl.includes('127.0.0.1')) {
+    throw new Error(
+      `DATABASE_URL must be localhost for test isolation. Got: ${dbUrl.replace(
+        /:[^:@]+@/,
+        ':***@',
+      )}`,
+    );
+  }
+
+  console.log(`[test] ${installed.length} installed store(s) in pool.`);
+  console.log(`[test] Test DB: ${dbUrl}`);
+  console.log(`[test] NODE_ENV: ${nodeEnv}`);
+
+  // ── Auth validity pre-flight ──────────────────────────────
+  // The checks above prove the storageState FILE exists and the registry
+  // is populated — but NOT that the captured Shopify session is still
+  // VALID. A stale session (or a rotated cf_clearance) makes every online
+  // test dead-end identically: openAppFrame navigates to admin.shopify.com,
+  // Shopify bounces to login (or Cloudflare shows Turnstile), the embedded
+  // app iframe never appears, and the test times out after 30s. With N
+  // tests that's N identical, cryptic "waiting for iframe[src*=localhost]"
+  // failures with no hint that the real cause is auth.
+  //
+  // So navigate ONCE here, exactly as the tests do, and abort the whole run
+  // with a clear, actionable message if the session is dead. A throw in
+  // globalSetup stops everything before any test starts — one message
+  // instead of N timeouts. Opt out with TEST_ONLINE_SKIP_AUTH_PREFLIGHT=true.
+  if ((process.env['TEST_ONLINE_SKIP_AUTH_PREFLIGHT'] ?? '').toLowerCase() !== 'true') {
+    const slug = (installed[0]?.['slug'] as string | undefined) ?? '';
+    if (slug) {
+      console.log('[test] auth pre-flight: verifying the captured session is still valid…');
+      await assertAuthValid(slug);
+      console.log('[test] auth pre-flight: session OK.');
+    }
+  }
+}
+
+/**
+ * Navigate to an installed store's embedded-app admin URL with the
+ * captured storageState — the SAME patchright launch the online tests use
+ * (storePool) — and classify the outcome:
+ *   - app iframe appears, or we reach the signed-in account picker → OK
+ *   - redirected to the Shopify login page → session expired
+ *   - Cloudflare Turnstile wall → cf_clearance stale
+ * Throws a clear "re-capture auth" message for the two failure modes so the
+ * whole run aborts with one diagnosis instead of N iframe-timeout failures.
+ */
+async function assertAuthValid(slug: string): Promise<void> {
+  const inContainer = (process.env['TEST_IN_CONTAINER'] ?? '').toLowerCase() === 'true';
+  const visible = (process.env['TEST_VISIBLE'] ?? '').toLowerCase() === 'true';
+  const headless = inContainer ? false : !visible;
+  const appHandle = process.env['SHOPIFY_APP_HANDLE'] ?? 'essential-app';
+  const expectedBackend = new URL(
+    process.env['SHOPIFY_APP_URL'] ?? 'https://localhost:8181',
+  ).hostname;
+  const url = `https://admin.shopify.com/store/${slug}/apps/${appHandle}`;
+
+  const fix =
+    `Re-run interactive auth capture (\`npm run test:online:auth\`): a browser opens — ` +
+    `sign in and pass Turnstile once, and the fresh session + cf_clearance are saved ` +
+    `back into storageState.json. Then re-run \`npm run test:online\`. ` +
+    `(Set TEST_ONLINE_SKIP_AUTH_PREFLIGHT=true to skip this check.)`;
+
+  const browser = await launchStealthBrowser({
+    headless,
+    hideWindow: !visible && !inContainer,
+  });
+  try {
+    const context = await browser.newContext({
+      storageState: storageStatePath,
+      viewport: { width: 1400, height: 900 },
+      ignoreHTTPSErrors: true,
+    });
+    const page = await context.newPage();
+    await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => {});
+
+    const deadline = Date.now() + 30_000;
+    while (Date.now() < deadline) {
+      const u = page.url();
+      // Cloudflare Turnstile wall (same probe text openApp uses).
+      if (
+        await page
+          .getByText('Your connection needs to be verified')
+          .isVisible()
+          .catch(() => false)
+      ) {
+        throw new Error(
+          `\n\n❌ Online auth pre-flight: Cloudflare Turnstile is blocking ${url}.\n` +
+            `   Your captured cf_clearance is missing or expired.\n   ${fix}\n`,
+        );
+      }
+      // Stale session → bounced to the Shopify login / account lookup.
+      // (accounts.shopify.com/select is the SIGNED-IN account picker — OK.)
+      if (
+        /accounts\.shopify\.com\/(lookup|signin|login|store_login|authentication)/.test(u) ||
+        /identity\.myshopify\.com/.test(u)
+      ) {
+        throw new Error(
+          `\n\n❌ Online auth pre-flight: redirected to the Shopify login page (${u}).\n` +
+            `   Your captured session has expired.\n   ${fix}\n`,
+        );
+      }
+      // Signed-in account picker — valid session.
+      if (u.includes('accounts.shopify.com/select')) return;
+      // Embedded app iframe present — signed in AND app reachable.
+      if ((await page.locator(`iframe[src*="${expectedBackend}"]`).count()) > 0) return;
+      await page.waitForTimeout(500);
+    }
+
+    // 30s without a definitive verdict. If parked on a login page, it's
+    // stale; otherwise it's likely a backend/app problem (NOT auth) — warn
+    // and let the per-test diagnostics speak rather than block on a guess.
+    const finalUrl = page.url();
+    if (/accounts\.shopify\.com/.test(finalUrl) && !finalUrl.includes('/select')) {
+      throw new Error(
+        `\n\n❌ Online auth pre-flight: stuck on the Shopify login page (${finalUrl}).\n   ${fix}\n`,
+      );
+    }
+    console.warn(
+      `[test] auth pre-flight inconclusive (last URL: ${finalUrl}). The session looks ` +
+        `valid (no login redirect / Turnstile) but the app iframe didn't appear within ` +
+        `30s — if tests fail on iframe load, check the app backend, not auth.`,
+    );
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}

package/src/playwright/index.ts

@@ -0,0 +1,11 @@
+/**
+ * Public Playwright preset for @essential-apps/shopify-test-runner.
+ * Imported by the consuming app's `playwright.config.ts`:
+ *
+ *   import { definePlaywrightConfig } from
+ *     '@essential-apps/shopify-test-runner/playwright';
+ */
+export {
+  definePlaywrightConfig,
+  type DefinePlaywrightConfigOptions,
+} from './baseConfig.js';